From 903406389ed86763065ebe9e7001b98082e909b5 Mon Sep 17 00:00:00 2001 From: XingY Date: Thu, 8 Jan 2026 17:09:04 -0800 Subject: [PATCH 01/14] StaleSampleFiles table for listing files no longer referenced by sample --- .../org/labkey/api/exp/query/ExpSchema.java | 1657 +++---- .../exp/query/ExpStaleSampleFilesTable.java | 101 + .../labkey/api/files/FileContentService.java | 722 +-- .../org/labkey/api/files/FileListener.java | 199 +- .../api/files/TableUpdaterFileListener.java | 912 ++-- .../experiment/FileLinkFileListener.java | 657 +-- .../filecontent/FileContentServiceImpl.java | 3927 +++++++++-------- 7 files changed, 4171 insertions(+), 4004 deletions(-) create mode 100644 api/src/org/labkey/api/exp/query/ExpStaleSampleFilesTable.java diff --git a/api/src/org/labkey/api/exp/query/ExpSchema.java b/api/src/org/labkey/api/exp/query/ExpSchema.java index 5fd2cedf774..72ffff090c6 100644 --- a/api/src/org/labkey/api/exp/query/ExpSchema.java +++ b/api/src/org/labkey/api/exp/query/ExpSchema.java @@ -1,824 +1,833 @@ -/* - * Copyright (c) 2009-2019 LabKey Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.labkey.api.exp.query; - -import org.jetbrains.annotations.NotNull; -import org.jetbrains.annotations.Nullable; -import org.labkey.api.collections.CaseInsensitiveHashSet; -import org.labkey.api.compliance.ComplianceService; -import org.labkey.api.data.ColumnInfo; -import org.labkey.api.data.Container; -import org.labkey.api.data.ContainerFilter; -import org.labkey.api.data.ContainerManager; -import org.labkey.api.data.EnumTableInfo; -import org.labkey.api.data.ForeignKey; -import org.labkey.api.data.TableInfo; -import org.labkey.api.data.UnionContainerFilter; -import org.labkey.api.exp.api.ExpRun; -import org.labkey.api.exp.api.ExpSampleType; -import org.labkey.api.exp.api.ExperimentService; -import org.labkey.api.exp.api.SampleTypeService; -import org.labkey.api.exp.property.DomainProperty; -import org.labkey.api.exp.property.Lookup; -import org.labkey.api.module.Module; -import org.labkey.api.ontology.KindOfQuantity; -import org.labkey.api.ontology.Unit; -import org.labkey.api.query.DefaultSchema; -import org.labkey.api.query.FieldKey; -import org.labkey.api.query.LookupForeignKey; -import org.labkey.api.query.QuerySchema; -import org.labkey.api.query.QuerySettings; -import org.labkey.api.query.QueryView; -import org.labkey.api.query.SchemaKey; -import org.labkey.api.security.User; -import org.labkey.api.security.permissions.InsertPermission; -import org.labkey.api.security.permissions.ReadPermission; -import org.labkey.api.util.StringExpression; -import org.labkey.api.view.ActionURL; -import org.labkey.api.view.ViewContext; -import org.springframework.validation.BindException; - -import java.util.ArrayList; -import java.util.Arrays; -import java.util.Collections; -import java.util.EnumSet; -import java.util.LinkedHashSet; -import java.util.List; -import java.util.Objects; -import java.util.Set; -import java.util.TreeSet; -import java.util.stream.Collectors; - -public class ExpSchema extends AbstractExpSchema -{ - public static final String EXPERIMENTS_MEMBERSHIP_FOR_RUN_TABLE_NAME = "ExperimentsMembershipForRun"; - public static final String DATA_CLASS_CATEGORY_TABLE = "DataClassCategoryType"; - public static final String SAMPLE_STATE_TYPE_TABLE = "SampleStateType"; - public static final String SAMPLE_TYPE_CATEGORY_TABLE = "SampleTypeCategoryType"; - public static final String MEASUREMENT_UNITS_TABLE = "MeasurementUnits"; - - public static final SchemaKey SCHEMA_EXP = SchemaKey.fromParts(ExpSchema.SCHEMA_NAME); - public static final SchemaKey SCHEMA_EXP_DATA = SchemaKey.fromString(SCHEMA_EXP, ExpSchema.NestedSchemas.data.name()); - public static final SchemaKey SCHEMA_EXP_MATERIALS = SchemaKey.fromString(SCHEMA_EXP, ExpSchema.NestedSchemas.materials.name()); - - private static final Set ADDITIONAL_SOURCES_AUDIT_FIELDS = new CaseInsensitiveHashSet("Name", "RowId"); - - public enum NestedSchemas - { - data, - materials - } - - public enum TableType - { - Runs - { - @Override - public TableInfo createTable(ExpSchema expSchema, String queryName, ContainerFilter cf) - { - ExpRunTable ret = ExperimentService.get().createRunTable(Runs.toString(), expSchema, cf); - return expSchema.setupTable(ret); - } - }, - Data - { - @Override - public TableInfo createTable(ExpSchema expSchema, String queryName, ContainerFilter cf) - { - ExpDataTable ret = ExperimentService.get().createDataTable(Data.toString(), expSchema, cf); - return expSchema.setupTable(ret); - } - }, - DataInputs - { - @Override - public TableInfo createTable(ExpSchema expSchema, String queryName, ContainerFilter cf) - { - ExpDataInputTable ret = ExperimentService.get().createDataInputTable(DataInputs.toString(), expSchema, cf); - return expSchema.setupTable(ret); - } - }, - DataProtocolInputs - { - @Override - public TableInfo createTable(ExpSchema expSchema, String queryName, ContainerFilter cf) - { - ExpDataProtocolInputTable ret = ExperimentService.get().createDataProtocolInputTable(DataProtocolInputs.toString(), expSchema, cf); - return expSchema.setupTable(ret); - } - }, - Materials - { - @Override - public TableInfo createTable(ExpSchema expSchema, String queryName, ContainerFilter cf) - { - ExpMaterialTable ret = ExperimentService.get().createMaterialTable(expSchema, cf, null); - return expSchema.setupTable(ret); - } - }, - MaterialInputs - { - @Override - public TableInfo createTable(ExpSchema expSchema, String queryName, ContainerFilter cf) - { - ExpMaterialInputTable ret = ExperimentService.get().createMaterialInputTable(MaterialInputs.toString(), expSchema, cf); - return expSchema.setupTable(ret); - } - }, - MaterialProtocolInputs - { - @Override - public TableInfo createTable(ExpSchema expSchema, String queryName, ContainerFilter cf) - { - ExpMaterialProtocolInputTable ret = ExperimentService.get().createMaterialProtocolInputTable(MaterialProtocolInputs.toString(), expSchema, cf); - return expSchema.setupTable(ret); - } - }, - Protocols - { - @Override - public TableInfo createTable(ExpSchema expSchema, String queryName, ContainerFilter cf) - { - ExpProtocolTable ret = ExperimentService.get().createProtocolTable(Protocols.toString(), expSchema, cf); - return expSchema.setupTable(ret); - } - }, - SampleSets - { - @Override - public TableInfo createTable(ExpSchema expSchema, String queryName, ContainerFilter cf) - { - ExpSampleTypeTable ret = ExperimentService.get().createSampleTypeTable(SampleSets.toString(), expSchema, cf); - return expSchema.setupTable(ret); - } - }, - DataClasses - { - @Override - public TableInfo createTable(ExpSchema expSchema, String queryName, ContainerFilter cf) - { - ExpDataClassTable ret = ExperimentService.get().createDataClassTable(DataClasses.toString(), expSchema, cf); - return expSchema.setupTable(ret); - } - }, - RunGroups - { - @Override - public TableInfo createTable(ExpSchema expSchema, String queryName, ContainerFilter cf) - { - ExpExperimentTable ret = ExperimentService.get().createExperimentTable(RunGroups.toString(), expSchema, cf); - return expSchema.setupTable(ret); - } - }, - RunGroupMap - { - @Override - public TableInfo createTable(ExpSchema expSchema, String queryName, ContainerFilter cf) - { - ExpRunGroupMapTable ret = ExperimentService.get().createRunGroupMapTable(RunGroupMap.toString(), expSchema, cf); - return expSchema.setupTable(ret); - } - }, - ProtocolApplications - { - @Override - public TableInfo createTable(ExpSchema expSchema, String queryName, ContainerFilter cf) - { - ExpProtocolApplicationTable result = ExperimentService.get().createProtocolApplicationTable(ProtocolApplications.toString(), expSchema, cf); - return expSchema.setupTable(result); - } - }, - QCFlags - { - @Override - public TableInfo createTable(ExpSchema expSchema, String queryName, ContainerFilter cf) - { - ExpQCFlagTable result = ExperimentService.get().createQCFlagsTable(QCFlags.toString(), expSchema, cf); - return expSchema.setupTable(result); - } - }, - Files - { - @Override - public TableInfo createTable(ExpSchema expSchema, String queryName, ContainerFilter cf) - { - ExpDataTable result = ExperimentService.get().createFilesTable(Files.toString(), expSchema); - return expSchema.setupTable(result); - } - }, - SampleStatus - { - @Override - public TableInfo createTable(ExpSchema expSchema, String queryName, ContainerFilter cf) - { - return ExperimentService.get().createSampleStatusTable(expSchema, cf); - } - }, - Fields - { - @Override - public TableInfo createTable(ExpSchema expSchema, String queryName, ContainerFilter cf) - { - return ExperimentService.get().createFieldsTable(expSchema, cf); - } - }, - PhiFields - { - @Override - public TableInfo createTable(ExpSchema expSchema, String queryName, ContainerFilter cf) - { - return ExperimentService.get().createPhiFieldsTable(expSchema, cf); - } - - @Override - public boolean includeTable() - { - return ComplianceService.get().isComplianceSupported(); - } - }; - public abstract TableInfo createTable(ExpSchema expSchema, String queryName, ContainerFilter cf); - - public boolean includeTable() - { - return true; - } - } - - public ExpTable getTable(TableType tableType) - { - return (ExpTable)getTable(tableType.toString()); - } - - public ExpTable getTable(TableType tableType, ContainerFilter cf) - { - return (ExpTable)getTable(tableType.toString(), cf); - } - - public ExpExperimentTable createExperimentsTableWithRunMemberships(ExpRun run, ContainerFilter cf) - { - ExpExperimentTable ret = ExperimentService.get().createExperimentTable(EXPERIMENTS_MEMBERSHIP_FOR_RUN_TABLE_NAME, this, cf); - setupTable(ret); - // Don't include exp.experiment rows that are assay batches - ret.setBatchProtocol(null); - ret.getMutableColumn(ExpExperimentTable.Column.RunCount).setHidden(true); - - ret.addExperimentMembershipColumn(run); - List defaultCols = new ArrayList<>(ret.getDefaultVisibleColumns()); - defaultCols.add(0, FieldKey.fromParts("RunMembership")); - defaultCols.remove(FieldKey.fromParts(ExpExperimentTable.Column.RunCount.name())); - ret.setDefaultVisibleColumns(defaultCols); - - return ret; - } - - // Use a holder pattern to initialize table names lazily, since ExpSchema gets referenced extremely early, before - // TableType.includeTable() might be ready to be called. e.g., ComplianceService isn't initialized yet. - private static final class TableNamesHolder - { - private static final Set tableNames; - - static - { - Set names = Arrays.stream(TableType.values()) - .filter(TableType::includeTable) - .map(TableType::toString) - .collect(Collectors.toCollection(LinkedHashSet::new)); - - names.add(DATA_CLASS_CATEGORY_TABLE); - names.add(SAMPLE_STATE_TYPE_TABLE); - names.add(SAMPLE_TYPE_CATEGORY_TABLE); - names.add(MEASUREMENT_UNITS_TABLE); - - tableNames = Collections.unmodifiableSet(names); - } - } - - public static final String SCHEMA_NAME = "exp"; - public static final String SCHEMA_DESCR = "Contains data about experiment runs, data files, materials, sample types, etc."; - - static public void register(final Module module) - { - DefaultSchema.registerProvider(SCHEMA_NAME, new DefaultSchema.SchemaProvider(module) - { - @Override - public boolean isAvailable(DefaultSchema schema, Module module) - { - return true; - } - - @Override - public QuerySchema createSchema(DefaultSchema schema, Module module) - { - return new ExpSchema(schema.getUser(), schema.getContainer()); - } - }); - } - - public SamplesSchema getSamplesSchema() - { - SamplesSchema schema = new SamplesSchema(this); - schema.setContainerFilter(_containerFilter); - return schema; - } - - public ExpSchema(User user, Container container) - { - super(SCHEMA_NAME, SCHEMA_DESCR, user, container, ExperimentService.get().getSchema()); - } - - @Override - public Set getTableNames() - { - return TableNamesHolder.tableNames; - } - - @Override - public TableInfo createTable(String name, ContainerFilter cf) - { - for (TableType tableType : TableType.values()) - { - if (tableType.name().equalsIgnoreCase(name) && tableType.includeTable()) - { - return tableType.createTable(this, tableType.name(), cf); - } - } - - if ("Experiments".equalsIgnoreCase(name)) - { - // Support "Experiments" as a legacy name for the RunGroups table - return TableType.RunGroups.createTable(this, name, cf); - } - if ("Datas".equalsIgnoreCase(name)) - { - /// Support "Datas" as a legacy name for the Data table - return TableType.Data.createTable(this, name, cf); - } - if (EXPERIMENTS_MEMBERSHIP_FOR_RUN_TABLE_NAME.equalsIgnoreCase(name)) - { - return createExperimentsTableWithRunMemberships(null, cf); - } - - if (DATA_CLASS_CATEGORY_TABLE.equalsIgnoreCase(name)) - { - return new EnumTableInfo<>(DataClassCategoryType.class, this, DataClassCategoryType::name, true, "Contains the list of available data class category types."); - } - - if (SAMPLE_STATE_TYPE_TABLE.equalsIgnoreCase(name)) - { - return new EnumTableInfo<>(SampleStateType.class, this, SampleStateType::name, true, "Contains the available sample state (status) types."); - } - - if (SAMPLE_TYPE_CATEGORY_TABLE.equalsIgnoreCase(name)) - { - return new EnumTableInfo<>(SampleTypeCategoryType.class, this, SampleTypeCategoryType::name, true, "Contains the list of available sample type category types."); - } - - if (MEASUREMENT_UNITS_TABLE.equalsIgnoreCase(name)) - { - // Create an EnumSet of the KindOfQuantity getCommonUnits - List commonUnits = KindOfQuantity.getSupportedUnits(); - EnumTableInfo table = new EnumTableInfo<>(Unit.class, EnumSet.copyOf(commonUnits), this, Unit::name, Unit::ordinal, false, "Contains the list of available units for measurements such as sample stored amounts."); - table.setPublicSchemaName(this.getName()); - table.setName(MEASUREMENT_UNITS_TABLE); - return table; - } - - return null; - } - - /** - * Exposed as EnumTableInfo. Update stateTypeEnum in qcStates.xsd if the enum values change. - */ - public enum SampleStateType - { - Available(Set.of(SampleTypeService.SampleOperations.values())), - Consumed(Set.of( - SampleTypeService.SampleOperations.EditMetadata, - SampleTypeService.SampleOperations.EditLineage, - SampleTypeService.SampleOperations.RemoveFromStorage, - SampleTypeService.SampleOperations.AddToPicklist, - SampleTypeService.SampleOperations.Delete, - SampleTypeService.SampleOperations.AddToWorkflow, - SampleTypeService.SampleOperations.RemoveFromWorkflow, - SampleTypeService.SampleOperations.AddAssayData, - SampleTypeService.SampleOperations.LinkToStudy, - SampleTypeService.SampleOperations.RecallFromStudy, - SampleTypeService.SampleOperations.Move - )), - Locked(Set.of( - SampleTypeService.SampleOperations.AddToPicklist - )); - - Set _permittedOps; - - SampleStateType(Set permittedOps) - { - _permittedOps = permittedOps; - } - - public Set getPermittedOps() - { - return _permittedOps; - } - } - - - /** - * Exposed as EnumTableInfo - * - */ - public enum DataClassCategoryType - { - registry(null), - media(null), - sources(ADDITIONAL_SOURCES_AUDIT_FIELDS); - - public final Set additionalAuditFields; - - DataClassCategoryType(@Nullable Set addlAuditFields) - { - this.additionalAuditFields = addlAuditFields; - } - - public static DataClassCategoryType fromString(String typeVal) { - for (DataClassCategoryType type : DataClassCategoryType.values()) { - if (type.name().equalsIgnoreCase(typeVal)) { - return type; - } - } - return null; - } - } - - public enum SampleTypeCategoryType - { - media; - - public static SampleTypeCategoryType fromString(String typeVal) { - for (SampleTypeCategoryType type : SampleTypeCategoryType.values()) { - if (type.name().equalsIgnoreCase(typeVal)) { - return type; - } - } - return null; - } - } - - @Override - public Set getSchemaNames() - { - if (_restricted) - return Collections.emptySet(); - - Set names = new TreeSet<>(String.CASE_INSENSITIVE_ORDER); - names.addAll(super.getSchemaNames()); - names.add(NestedSchemas.materials.name()); - names.add(NestedSchemas.data.name()); - return names; - } - - public QuerySchema getSchema(NestedSchemas schema) - { - return getSchema(schema.name()); - } - - @Override - public QuerySchema getSchema(String name) - { - if (_restricted) - return null; - - // CONSIDER: also support hidden "samples" schema ? - if (name.equals(NestedSchemas.materials.name())) - return new SamplesSchema(SchemaKey.fromParts(getName(), NestedSchemas.materials.name()), getUser(), getContainer()); - - if (name.equals(NestedSchemas.data.name())) - return new DataClassUserSchema(getContainer(), getUser()); - - return super.getSchema(name); - } - - public ExpDataTable getDatasTable() - { - return (ExpDataTable)getTable(TableType.Data); - } - - public ExpDataTable getDatasTable(boolean forWrite) - { - return (ExpDataTable)getTable(TableType.Data.toString(), null, true, forWrite); - } - - public ExpRunTable getRunsTable() - { - return (ExpRunTable)getTable(TableType.Runs.toString(), null, true, false); - } - - public ExpRunTable getRunsTable(boolean forWrite) - { - return (ExpRunTable)getTable(TableType.Runs.toString(), null, true, forWrite); - } - - - public ForeignKey getProtocolApplicationForeignKey(ContainerFilter cf) - { - return new ExperimentLookupForeignKey(null, null, ExpSchema.SCHEMA_NAME, TableType.ProtocolApplications.name(), "RowId", null) - { - @Override - public TableInfo getLookupTableInfo() - { - return getTable(TableType.ProtocolApplications, cf); - } - }; - } - - public ForeignKey getProtocolForeignKey(ContainerFilter cf, String targetColumnName) - { - return new LookupForeignKey(targetColumnName) - { - @Override - public TableInfo getLookupTableInfo() - { - return getTable(TableType.Protocols.toString(), ContainerFilter.getUnsafeEverythingFilter()); - } - }; - } - - public ForeignKey getMaterialForeignKey(ContainerFilter cf, String targetColumnName) - { - return new LookupForeignKey(targetColumnName) - { - @Override - public TableInfo getLookupTableInfo() - { - return getTable(TableType.Materials.toString(), cf); - } - - @Override - public StringExpression getURL(ColumnInfo parent) - { - return getURL(parent, true); - } - }; - } - - public ForeignKey getMaterialProtocolInputForeignKey(ContainerFilter cf) - { - return new ExperimentLookupForeignKey(null, null, ExpSchema.SCHEMA_NAME, TableType.MaterialProtocolInputs.name(), "RowId", null) - { - @Override - public TableInfo getLookupTableInfo() - { - return getTable(TableType.MaterialProtocolInputs, cf); - } - }; - } - - public ForeignKey getDataProtocolInputForeignKey(ContainerFilter cf) - { - return new ExperimentLookupForeignKey(null, null, ExpSchema.SCHEMA_NAME, TableType.DataProtocolInputs.name(), "RowId", null) - { - @Override - public TableInfo getLookupTableInfo() - { - return getTable(TableType.DataProtocolInputs, cf); - } - }; - } - - public ForeignKey getJobForeignKey() - { - return new LookupForeignKey("RowId", "RowId") - { - @Override - public TableInfo getLookupTableInfo() - { - QuerySchema pipeline = getDefaultSchema().getSchema("pipeline"); - if (null == pipeline) - return null; - return pipeline.getTable("Job", getDefaultContainerFilter()); - } - - @Override - public StringExpression getURL(ColumnInfo parent) - { - return getURL(parent, true); - } - }; - } - - @Deprecated - public ForeignKey getRunIdForeignKey() - { - return getRunIdForeignKey(null); - } - - public ForeignKey getRunIdForeignKey(ContainerFilter cf) - { - return new ExperimentLookupForeignKey(null, null, ExpSchema.SCHEMA_NAME, TableType.Runs.name(), "RowId", null) - { - @Override - public TableInfo getLookupTableInfo() - { - return getTable(TableType.Runs, cf); - } - }; - } - - @Deprecated - public ForeignKey getRunGroupIdForeignKey(final boolean includeBatches) - { - return getRunGroupIdForeignKey(null, includeBatches); - } - - /** @param includeBatches if false, then filter out run groups of type batch when doing the join */ - public ForeignKey getRunGroupIdForeignKey(ContainerFilter cf, final boolean includeBatches) - { - return new ExperimentLookupForeignKey(null, null, ExpSchema.SCHEMA_NAME, TableType.RunGroups.name(), "RowId", null) - { - @Override - public TableInfo getLookupTableInfo() - { - ContainerFilter cf = getLookupContainerFilter(); - String key = getClass().getName() + "/RunGroupIdForeignKey/" + includeBatches + "/" + cf.getCacheKey(); - // since getTable(forWrite=true) does not cache, cache this tableinfo using getCachedLookupTableInfo() - return ExpSchema.this.getCachedLookupTableInfo(key, this::createLookupTableInfo); - } - - @Override - protected ContainerFilter getLookupContainerFilter() - { - return Objects.requireNonNullElse(cf, ContainerFilter.Type.CurrentPlusProjectAndShared.create(ExpSchema.this)); - } - - private TableInfo createLookupTableInfo() - { - // CONSIDER: I wonder if this shouldn't be using UnionContainerFilter(cf, CurrentPlusProjectAndShared) - ExpExperimentTable result = (ExpExperimentTable) getTable(TableType.RunGroups.name(), getLookupContainerFilter(), true, true); - if (!includeBatches) - { - result.setBatchProtocol(null); - } - result.setLocked(true); - return result; - } - }; - } - - public ForeignKey getDataIdForeignKey(ContainerFilter cf) - { - return new ExperimentLookupForeignKey(null, null, ExpSchema.SCHEMA_NAME, TableType.Data.name(), "RowId", null) - { - @Override - public TableInfo getLookupTableInfo() - { - return getTable(TableType.Data, cf); - } - }; - } - - /** - * @param domainProperty the property on which the lookup is configured - */ - @NotNull - public ForeignKey getMaterialIdForeignKey(@Nullable ExpSampleType targetSampleType, @Nullable DomainProperty domainProperty, @Nullable ContainerFilter cfParent) - { - if (targetSampleType == null) - { - return new ExperimentLookupForeignKey(null, null, ExpSchema.SCHEMA_NAME, TableType.Materials.name(), "RowId", null) - { - @Override - public TableInfo getLookupTableInfo() - { - ContainerFilter cf = new ContainerFilter.SimpleContainerFilter(getSearchContainers(getContainer(), targetSampleType, domainProperty, getUser())); - if (null != cfParent) - cf = new UnionContainerFilter(cf, cfParent); - ExpTable result = getTable(TableType.Materials, cf); - return result; - } - }; - } - return getSamplesSchema().materialIdForeignKey(targetSampleType, domainProperty, cfParent); - } - - @NotNull - public static Set getSearchContainers(Container currentContainer, @Nullable ExpSampleType st, @Nullable DomainProperty dp, User user) - { - Set searchContainers = new LinkedHashSet<>(); - if (dp != null) - { - Lookup lookup = dp.getLookup(); - if (lookup != null && lookup.getContainer() != null) - { - Container lookupContainer = lookup.getContainer(); - if (lookupContainer.hasPermission(user, ReadPermission.class)) - { - // The property is specifically targeting a container, so look there and only there - searchContainers.add(lookup.getContainer()); - } - } - } - - if (searchContainers.isEmpty()) - { - // Default to looking in the current container - searchContainers.add(currentContainer); - if (st == null || (st.getContainer().isProject() && !currentContainer.isProject())) - { - Container c = currentContainer.getParent(); - // Recurse up the chain to the project - while (c != null && !c.isRoot()) - { - if (c.hasPermission(user, ReadPermission.class)) - { - searchContainers.add(c); - } - c = c.getParent(); - } - } - Container sharedContainer = ContainerManager.getSharedContainer(); - if (st == null || st.getContainer().equals(sharedContainer)) - { - if (sharedContainer.hasPermission(user, ReadPermission.class)) - { - searchContainers.add(ContainerManager.getSharedContainer()); - } - } - } - return searchContainers; - } - - public abstract static class ExperimentLookupForeignKey extends LookupForeignKey - { - public ExperimentLookupForeignKey(String pkColumnName) - { - super(pkColumnName); - } - - public ExperimentLookupForeignKey(ActionURL baseURL, String paramName, String schemaName, String tableName, String pkColumnName, String titleColumn) - { - super(baseURL, paramName, schemaName, tableName, pkColumnName, titleColumn); - } - - @Override - public StringExpression getURL(ColumnInfo parent) - { - return getURL(parent, true); - } - } - - @Override - public @NotNull QueryView createView(ViewContext context, @NotNull QuerySettings settings, BindException errors) - { - if (TableType.DataClasses.name().equalsIgnoreCase(settings.getQueryName())) - { - return new QueryView(this, settings, errors) - { - @Override - protected boolean canInsert() - { - TableInfo table = getTable(); - return table != null && table.hasPermission(getUser(), InsertPermission.class); - } - - @Override - public boolean showImportDataButton() - { - return false; - } - }; - } - - QueryView queryView = super.createView(context, settings, errors); - - if (TableType.Materials.name().equalsIgnoreCase(settings.getQueryName()) || - TableType.Data.name().equalsIgnoreCase(settings.getQueryName()) || - TableType.Protocols.name().equalsIgnoreCase(settings.getQueryName())) - { - // Use default delete button, but without showing the confirmation text - queryView.setShowDeleteButtonConfirmationText(false); - } - - return queryView; - } - - public enum DerivationDataScopeType - { - ChildOnly, - ParentOnly, - All - } -} +/* + * Copyright (c) 2009-2019 LabKey Corporation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.labkey.api.exp.query; + +import org.jetbrains.annotations.NotNull; +import org.jetbrains.annotations.Nullable; +import org.labkey.api.collections.CaseInsensitiveHashSet; +import org.labkey.api.compliance.ComplianceService; +import org.labkey.api.data.ColumnInfo; +import org.labkey.api.data.Container; +import org.labkey.api.data.ContainerFilter; +import org.labkey.api.data.ContainerManager; +import org.labkey.api.data.EnumTableInfo; +import org.labkey.api.data.ForeignKey; +import org.labkey.api.data.TableInfo; +import org.labkey.api.data.UnionContainerFilter; +import org.labkey.api.exp.api.ExpRun; +import org.labkey.api.exp.api.ExpSampleType; +import org.labkey.api.exp.api.ExperimentService; +import org.labkey.api.exp.api.SampleTypeService; +import org.labkey.api.exp.property.DomainProperty; +import org.labkey.api.exp.property.Lookup; +import org.labkey.api.module.Module; +import org.labkey.api.ontology.KindOfQuantity; +import org.labkey.api.ontology.Unit; +import org.labkey.api.query.DefaultSchema; +import org.labkey.api.query.FieldKey; +import org.labkey.api.query.LookupForeignKey; +import org.labkey.api.query.QuerySchema; +import org.labkey.api.query.QuerySettings; +import org.labkey.api.query.QueryView; +import org.labkey.api.query.SchemaKey; +import org.labkey.api.security.User; +import org.labkey.api.security.permissions.InsertPermission; +import org.labkey.api.security.permissions.ReadPermission; +import org.labkey.api.util.StringExpression; +import org.labkey.api.view.ActionURL; +import org.labkey.api.view.ViewContext; +import org.springframework.validation.BindException; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.EnumSet; +import java.util.LinkedHashSet; +import java.util.List; +import java.util.Objects; +import java.util.Set; +import java.util.TreeSet; +import java.util.stream.Collectors; + +public class ExpSchema extends AbstractExpSchema +{ + public static final String EXPERIMENTS_MEMBERSHIP_FOR_RUN_TABLE_NAME = "ExperimentsMembershipForRun"; + public static final String DATA_CLASS_CATEGORY_TABLE = "DataClassCategoryType"; + public static final String SAMPLE_STATE_TYPE_TABLE = "SampleStateType"; + public static final String SAMPLE_TYPE_CATEGORY_TABLE = "SampleTypeCategoryType"; + public static final String MEASUREMENT_UNITS_TABLE = "MeasurementUnits"; + public static final String SAMPLE_FILES_TABLE = "StaleSampleFiles"; + + public static final SchemaKey SCHEMA_EXP = SchemaKey.fromParts(ExpSchema.SCHEMA_NAME); + public static final SchemaKey SCHEMA_EXP_DATA = SchemaKey.fromString(SCHEMA_EXP, ExpSchema.NestedSchemas.data.name()); + public static final SchemaKey SCHEMA_EXP_MATERIALS = SchemaKey.fromString(SCHEMA_EXP, ExpSchema.NestedSchemas.materials.name()); + + private static final Set ADDITIONAL_SOURCES_AUDIT_FIELDS = new CaseInsensitiveHashSet("Name", "RowId"); + + public enum NestedSchemas + { + data, + materials + } + + public enum TableType + { + Runs + { + @Override + public TableInfo createTable(ExpSchema expSchema, String queryName, ContainerFilter cf) + { + ExpRunTable ret = ExperimentService.get().createRunTable(Runs.toString(), expSchema, cf); + return expSchema.setupTable(ret); + } + }, + Data + { + @Override + public TableInfo createTable(ExpSchema expSchema, String queryName, ContainerFilter cf) + { + ExpDataTable ret = ExperimentService.get().createDataTable(Data.toString(), expSchema, cf); + return expSchema.setupTable(ret); + } + }, + DataInputs + { + @Override + public TableInfo createTable(ExpSchema expSchema, String queryName, ContainerFilter cf) + { + ExpDataInputTable ret = ExperimentService.get().createDataInputTable(DataInputs.toString(), expSchema, cf); + return expSchema.setupTable(ret); + } + }, + DataProtocolInputs + { + @Override + public TableInfo createTable(ExpSchema expSchema, String queryName, ContainerFilter cf) + { + ExpDataProtocolInputTable ret = ExperimentService.get().createDataProtocolInputTable(DataProtocolInputs.toString(), expSchema, cf); + return expSchema.setupTable(ret); + } + }, + Materials + { + @Override + public TableInfo createTable(ExpSchema expSchema, String queryName, ContainerFilter cf) + { + ExpMaterialTable ret = ExperimentService.get().createMaterialTable(expSchema, cf, null); + return expSchema.setupTable(ret); + } + }, + MaterialInputs + { + @Override + public TableInfo createTable(ExpSchema expSchema, String queryName, ContainerFilter cf) + { + ExpMaterialInputTable ret = ExperimentService.get().createMaterialInputTable(MaterialInputs.toString(), expSchema, cf); + return expSchema.setupTable(ret); + } + }, + MaterialProtocolInputs + { + @Override + public TableInfo createTable(ExpSchema expSchema, String queryName, ContainerFilter cf) + { + ExpMaterialProtocolInputTable ret = ExperimentService.get().createMaterialProtocolInputTable(MaterialProtocolInputs.toString(), expSchema, cf); + return expSchema.setupTable(ret); + } + }, + Protocols + { + @Override + public TableInfo createTable(ExpSchema expSchema, String queryName, ContainerFilter cf) + { + ExpProtocolTable ret = ExperimentService.get().createProtocolTable(Protocols.toString(), expSchema, cf); + return expSchema.setupTable(ret); + } + }, + SampleSets + { + @Override + public TableInfo createTable(ExpSchema expSchema, String queryName, ContainerFilter cf) + { + ExpSampleTypeTable ret = ExperimentService.get().createSampleTypeTable(SampleSets.toString(), expSchema, cf); + return expSchema.setupTable(ret); + } + }, + DataClasses + { + @Override + public TableInfo createTable(ExpSchema expSchema, String queryName, ContainerFilter cf) + { + ExpDataClassTable ret = ExperimentService.get().createDataClassTable(DataClasses.toString(), expSchema, cf); + return expSchema.setupTable(ret); + } + }, + RunGroups + { + @Override + public TableInfo createTable(ExpSchema expSchema, String queryName, ContainerFilter cf) + { + ExpExperimentTable ret = ExperimentService.get().createExperimentTable(RunGroups.toString(), expSchema, cf); + return expSchema.setupTable(ret); + } + }, + RunGroupMap + { + @Override + public TableInfo createTable(ExpSchema expSchema, String queryName, ContainerFilter cf) + { + ExpRunGroupMapTable ret = ExperimentService.get().createRunGroupMapTable(RunGroupMap.toString(), expSchema, cf); + return expSchema.setupTable(ret); + } + }, + ProtocolApplications + { + @Override + public TableInfo createTable(ExpSchema expSchema, String queryName, ContainerFilter cf) + { + ExpProtocolApplicationTable result = ExperimentService.get().createProtocolApplicationTable(ProtocolApplications.toString(), expSchema, cf); + return expSchema.setupTable(result); + } + }, + QCFlags + { + @Override + public TableInfo createTable(ExpSchema expSchema, String queryName, ContainerFilter cf) + { + ExpQCFlagTable result = ExperimentService.get().createQCFlagsTable(QCFlags.toString(), expSchema, cf); + return expSchema.setupTable(result); + } + }, + Files + { + @Override + public TableInfo createTable(ExpSchema expSchema, String queryName, ContainerFilter cf) + { + ExpDataTable result = ExperimentService.get().createFilesTable(Files.toString(), expSchema); + return expSchema.setupTable(result); + } + }, + StaleSampleFiles + { + @Override + public TableInfo createTable(ExpSchema expSchema, String queryName, ContainerFilter cf) + { + return new ExpStaleSampleFilesTable(expSchema, cf); + } + }, + SampleStatus + { + @Override + public TableInfo createTable(ExpSchema expSchema, String queryName, ContainerFilter cf) + { + return ExperimentService.get().createSampleStatusTable(expSchema, cf); + } + }, + Fields + { + @Override + public TableInfo createTable(ExpSchema expSchema, String queryName, ContainerFilter cf) + { + return ExperimentService.get().createFieldsTable(expSchema, cf); + } + }, + PhiFields + { + @Override + public TableInfo createTable(ExpSchema expSchema, String queryName, ContainerFilter cf) + { + return ExperimentService.get().createPhiFieldsTable(expSchema, cf); + } + + @Override + public boolean includeTable() + { + return ComplianceService.get().isComplianceSupported(); + } + }; + public abstract TableInfo createTable(ExpSchema expSchema, String queryName, ContainerFilter cf); + + public boolean includeTable() + { + return true; + } + } + + public ExpTable getTable(TableType tableType) + { + return (ExpTable)getTable(tableType.toString()); + } + + public ExpTable getTable(TableType tableType, ContainerFilter cf) + { + return (ExpTable)getTable(tableType.toString(), cf); + } + + public ExpExperimentTable createExperimentsTableWithRunMemberships(ExpRun run, ContainerFilter cf) + { + ExpExperimentTable ret = ExperimentService.get().createExperimentTable(EXPERIMENTS_MEMBERSHIP_FOR_RUN_TABLE_NAME, this, cf); + setupTable(ret); + // Don't include exp.experiment rows that are assay batches + ret.setBatchProtocol(null); + ret.getMutableColumn(ExpExperimentTable.Column.RunCount).setHidden(true); + + ret.addExperimentMembershipColumn(run); + List defaultCols = new ArrayList<>(ret.getDefaultVisibleColumns()); + defaultCols.add(0, FieldKey.fromParts("RunMembership")); + defaultCols.remove(FieldKey.fromParts(ExpExperimentTable.Column.RunCount.name())); + ret.setDefaultVisibleColumns(defaultCols); + + return ret; + } + + // Use a holder pattern to initialize table names lazily, since ExpSchema gets referenced extremely early, before + // TableType.includeTable() might be ready to be called. e.g., ComplianceService isn't initialized yet. + private static final class TableNamesHolder + { + private static final Set tableNames; + + static + { + Set names = Arrays.stream(TableType.values()) + .filter(TableType::includeTable) + .map(TableType::toString) + .collect(Collectors.toCollection(LinkedHashSet::new)); + + names.add(DATA_CLASS_CATEGORY_TABLE); + names.add(SAMPLE_STATE_TYPE_TABLE); + names.add(SAMPLE_TYPE_CATEGORY_TABLE); + names.add(MEASUREMENT_UNITS_TABLE); + + tableNames = Collections.unmodifiableSet(names); + } + } + + public static final String SCHEMA_NAME = "exp"; + public static final String SCHEMA_DESCR = "Contains data about experiment runs, data files, materials, sample types, etc."; + + static public void register(final Module module) + { + DefaultSchema.registerProvider(SCHEMA_NAME, new DefaultSchema.SchemaProvider(module) + { + @Override + public boolean isAvailable(DefaultSchema schema, Module module) + { + return true; + } + + @Override + public QuerySchema createSchema(DefaultSchema schema, Module module) + { + return new ExpSchema(schema.getUser(), schema.getContainer()); + } + }); + } + + public SamplesSchema getSamplesSchema() + { + SamplesSchema schema = new SamplesSchema(this); + schema.setContainerFilter(_containerFilter); + return schema; + } + + public ExpSchema(User user, Container container) + { + super(SCHEMA_NAME, SCHEMA_DESCR, user, container, ExperimentService.get().getSchema()); + } + + @Override + public Set getTableNames() + { + return TableNamesHolder.tableNames; + } + + @Override + public TableInfo createTable(String name, ContainerFilter cf) + { + for (TableType tableType : TableType.values()) + { + if (tableType.name().equalsIgnoreCase(name) && tableType.includeTable()) + { + return tableType.createTable(this, tableType.name(), cf); + } + } + + if ("Experiments".equalsIgnoreCase(name)) + { + // Support "Experiments" as a legacy name for the RunGroups table + return TableType.RunGroups.createTable(this, name, cf); + } + if ("Datas".equalsIgnoreCase(name)) + { + /// Support "Datas" as a legacy name for the Data table + return TableType.Data.createTable(this, name, cf); + } + if (EXPERIMENTS_MEMBERSHIP_FOR_RUN_TABLE_NAME.equalsIgnoreCase(name)) + { + return createExperimentsTableWithRunMemberships(null, cf); + } + + if (DATA_CLASS_CATEGORY_TABLE.equalsIgnoreCase(name)) + { + return new EnumTableInfo<>(DataClassCategoryType.class, this, DataClassCategoryType::name, true, "Contains the list of available data class category types."); + } + + if (SAMPLE_STATE_TYPE_TABLE.equalsIgnoreCase(name)) + { + return new EnumTableInfo<>(SampleStateType.class, this, SampleStateType::name, true, "Contains the available sample state (status) types."); + } + + if (SAMPLE_TYPE_CATEGORY_TABLE.equalsIgnoreCase(name)) + { + return new EnumTableInfo<>(SampleTypeCategoryType.class, this, SampleTypeCategoryType::name, true, "Contains the list of available sample type category types."); + } + + if (MEASUREMENT_UNITS_TABLE.equalsIgnoreCase(name)) + { + // Create an EnumSet of the KindOfQuantity getCommonUnits + List commonUnits = KindOfQuantity.getSupportedUnits(); + EnumTableInfo table = new EnumTableInfo<>(Unit.class, EnumSet.copyOf(commonUnits), this, Unit::name, Unit::ordinal, false, "Contains the list of available units for measurements such as sample stored amounts."); + table.setPublicSchemaName(this.getName()); + table.setName(MEASUREMENT_UNITS_TABLE); + return table; + } + + return null; + } + + /** + * Exposed as EnumTableInfo. Update stateTypeEnum in qcStates.xsd if the enum values change. + */ + public enum SampleStateType + { + Available(Set.of(SampleTypeService.SampleOperations.values())), + Consumed(Set.of( + SampleTypeService.SampleOperations.EditMetadata, + SampleTypeService.SampleOperations.EditLineage, + SampleTypeService.SampleOperations.RemoveFromStorage, + SampleTypeService.SampleOperations.AddToPicklist, + SampleTypeService.SampleOperations.Delete, + SampleTypeService.SampleOperations.AddToWorkflow, + SampleTypeService.SampleOperations.RemoveFromWorkflow, + SampleTypeService.SampleOperations.AddAssayData, + SampleTypeService.SampleOperations.LinkToStudy, + SampleTypeService.SampleOperations.RecallFromStudy, + SampleTypeService.SampleOperations.Move + )), + Locked(Set.of( + SampleTypeService.SampleOperations.AddToPicklist + )); + + Set _permittedOps; + + SampleStateType(Set permittedOps) + { + _permittedOps = permittedOps; + } + + public Set getPermittedOps() + { + return _permittedOps; + } + } + + + /** + * Exposed as EnumTableInfo + * + */ + public enum DataClassCategoryType + { + registry(null), + media(null), + sources(ADDITIONAL_SOURCES_AUDIT_FIELDS); + + public final Set additionalAuditFields; + + DataClassCategoryType(@Nullable Set addlAuditFields) + { + this.additionalAuditFields = addlAuditFields; + } + + public static DataClassCategoryType fromString(String typeVal) { + for (DataClassCategoryType type : DataClassCategoryType.values()) { + if (type.name().equalsIgnoreCase(typeVal)) { + return type; + } + } + return null; + } + } + + public enum SampleTypeCategoryType + { + media; + + public static SampleTypeCategoryType fromString(String typeVal) { + for (SampleTypeCategoryType type : SampleTypeCategoryType.values()) { + if (type.name().equalsIgnoreCase(typeVal)) { + return type; + } + } + return null; + } + } + + @Override + public Set getSchemaNames() + { + if (_restricted) + return Collections.emptySet(); + + Set names = new TreeSet<>(String.CASE_INSENSITIVE_ORDER); + names.addAll(super.getSchemaNames()); + names.add(NestedSchemas.materials.name()); + names.add(NestedSchemas.data.name()); + return names; + } + + public QuerySchema getSchema(NestedSchemas schema) + { + return getSchema(schema.name()); + } + + @Override + public QuerySchema getSchema(String name) + { + if (_restricted) + return null; + + // CONSIDER: also support hidden "samples" schema ? + if (name.equals(NestedSchemas.materials.name())) + return new SamplesSchema(SchemaKey.fromParts(getName(), NestedSchemas.materials.name()), getUser(), getContainer()); + + if (name.equals(NestedSchemas.data.name())) + return new DataClassUserSchema(getContainer(), getUser()); + + return super.getSchema(name); + } + + public ExpDataTable getDatasTable() + { + return (ExpDataTable)getTable(TableType.Data); + } + + public ExpDataTable getDatasTable(boolean forWrite) + { + return (ExpDataTable)getTable(TableType.Data.toString(), null, true, forWrite); + } + + public ExpRunTable getRunsTable() + { + return (ExpRunTable)getTable(TableType.Runs.toString(), null, true, false); + } + + public ExpRunTable getRunsTable(boolean forWrite) + { + return (ExpRunTable)getTable(TableType.Runs.toString(), null, true, forWrite); + } + + + public ForeignKey getProtocolApplicationForeignKey(ContainerFilter cf) + { + return new ExperimentLookupForeignKey(null, null, ExpSchema.SCHEMA_NAME, TableType.ProtocolApplications.name(), "RowId", null) + { + @Override + public TableInfo getLookupTableInfo() + { + return getTable(TableType.ProtocolApplications, cf); + } + }; + } + + public ForeignKey getProtocolForeignKey(ContainerFilter cf, String targetColumnName) + { + return new LookupForeignKey(targetColumnName) + { + @Override + public TableInfo getLookupTableInfo() + { + return getTable(TableType.Protocols.toString(), ContainerFilter.getUnsafeEverythingFilter()); + } + }; + } + + public ForeignKey getMaterialForeignKey(ContainerFilter cf, String targetColumnName) + { + return new LookupForeignKey(targetColumnName) + { + @Override + public TableInfo getLookupTableInfo() + { + return getTable(TableType.Materials.toString(), cf); + } + + @Override + public StringExpression getURL(ColumnInfo parent) + { + return getURL(parent, true); + } + }; + } + + public ForeignKey getMaterialProtocolInputForeignKey(ContainerFilter cf) + { + return new ExperimentLookupForeignKey(null, null, ExpSchema.SCHEMA_NAME, TableType.MaterialProtocolInputs.name(), "RowId", null) + { + @Override + public TableInfo getLookupTableInfo() + { + return getTable(TableType.MaterialProtocolInputs, cf); + } + }; + } + + public ForeignKey getDataProtocolInputForeignKey(ContainerFilter cf) + { + return new ExperimentLookupForeignKey(null, null, ExpSchema.SCHEMA_NAME, TableType.DataProtocolInputs.name(), "RowId", null) + { + @Override + public TableInfo getLookupTableInfo() + { + return getTable(TableType.DataProtocolInputs, cf); + } + }; + } + + public ForeignKey getJobForeignKey() + { + return new LookupForeignKey("RowId", "RowId") + { + @Override + public TableInfo getLookupTableInfo() + { + QuerySchema pipeline = getDefaultSchema().getSchema("pipeline"); + if (null == pipeline) + return null; + return pipeline.getTable("Job", getDefaultContainerFilter()); + } + + @Override + public StringExpression getURL(ColumnInfo parent) + { + return getURL(parent, true); + } + }; + } + + @Deprecated + public ForeignKey getRunIdForeignKey() + { + return getRunIdForeignKey(null); + } + + public ForeignKey getRunIdForeignKey(ContainerFilter cf) + { + return new ExperimentLookupForeignKey(null, null, ExpSchema.SCHEMA_NAME, TableType.Runs.name(), "RowId", null) + { + @Override + public TableInfo getLookupTableInfo() + { + return getTable(TableType.Runs, cf); + } + }; + } + + @Deprecated + public ForeignKey getRunGroupIdForeignKey(final boolean includeBatches) + { + return getRunGroupIdForeignKey(null, includeBatches); + } + + /** @param includeBatches if false, then filter out run groups of type batch when doing the join */ + public ForeignKey getRunGroupIdForeignKey(ContainerFilter cf, final boolean includeBatches) + { + return new ExperimentLookupForeignKey(null, null, ExpSchema.SCHEMA_NAME, TableType.RunGroups.name(), "RowId", null) + { + @Override + public TableInfo getLookupTableInfo() + { + ContainerFilter cf = getLookupContainerFilter(); + String key = getClass().getName() + "/RunGroupIdForeignKey/" + includeBatches + "/" + cf.getCacheKey(); + // since getTable(forWrite=true) does not cache, cache this tableinfo using getCachedLookupTableInfo() + return ExpSchema.this.getCachedLookupTableInfo(key, this::createLookupTableInfo); + } + + @Override + protected ContainerFilter getLookupContainerFilter() + { + return Objects.requireNonNullElse(cf, ContainerFilter.Type.CurrentPlusProjectAndShared.create(ExpSchema.this)); + } + + private TableInfo createLookupTableInfo() + { + // CONSIDER: I wonder if this shouldn't be using UnionContainerFilter(cf, CurrentPlusProjectAndShared) + ExpExperimentTable result = (ExpExperimentTable) getTable(TableType.RunGroups.name(), getLookupContainerFilter(), true, true); + if (!includeBatches) + { + result.setBatchProtocol(null); + } + result.setLocked(true); + return result; + } + }; + } + + public ForeignKey getDataIdForeignKey(ContainerFilter cf) + { + return new ExperimentLookupForeignKey(null, null, ExpSchema.SCHEMA_NAME, TableType.Data.name(), "RowId", null) + { + @Override + public TableInfo getLookupTableInfo() + { + return getTable(TableType.Data, cf); + } + }; + } + + /** + * @param domainProperty the property on which the lookup is configured + */ + @NotNull + public ForeignKey getMaterialIdForeignKey(@Nullable ExpSampleType targetSampleType, @Nullable DomainProperty domainProperty, @Nullable ContainerFilter cfParent) + { + if (targetSampleType == null) + { + return new ExperimentLookupForeignKey(null, null, ExpSchema.SCHEMA_NAME, TableType.Materials.name(), "RowId", null) + { + @Override + public TableInfo getLookupTableInfo() + { + ContainerFilter cf = new ContainerFilter.SimpleContainerFilter(getSearchContainers(getContainer(), targetSampleType, domainProperty, getUser())); + if (null != cfParent) + cf = new UnionContainerFilter(cf, cfParent); + ExpTable result = getTable(TableType.Materials, cf); + return result; + } + }; + } + return getSamplesSchema().materialIdForeignKey(targetSampleType, domainProperty, cfParent); + } + + @NotNull + public static Set getSearchContainers(Container currentContainer, @Nullable ExpSampleType st, @Nullable DomainProperty dp, User user) + { + Set searchContainers = new LinkedHashSet<>(); + if (dp != null) + { + Lookup lookup = dp.getLookup(); + if (lookup != null && lookup.getContainer() != null) + { + Container lookupContainer = lookup.getContainer(); + if (lookupContainer.hasPermission(user, ReadPermission.class)) + { + // The property is specifically targeting a container, so look there and only there + searchContainers.add(lookup.getContainer()); + } + } + } + + if (searchContainers.isEmpty()) + { + // Default to looking in the current container + searchContainers.add(currentContainer); + if (st == null || (st.getContainer().isProject() && !currentContainer.isProject())) + { + Container c = currentContainer.getParent(); + // Recurse up the chain to the project + while (c != null && !c.isRoot()) + { + if (c.hasPermission(user, ReadPermission.class)) + { + searchContainers.add(c); + } + c = c.getParent(); + } + } + Container sharedContainer = ContainerManager.getSharedContainer(); + if (st == null || st.getContainer().equals(sharedContainer)) + { + if (sharedContainer.hasPermission(user, ReadPermission.class)) + { + searchContainers.add(ContainerManager.getSharedContainer()); + } + } + } + return searchContainers; + } + + public abstract static class ExperimentLookupForeignKey extends LookupForeignKey + { + public ExperimentLookupForeignKey(String pkColumnName) + { + super(pkColumnName); + } + + public ExperimentLookupForeignKey(ActionURL baseURL, String paramName, String schemaName, String tableName, String pkColumnName, String titleColumn) + { + super(baseURL, paramName, schemaName, tableName, pkColumnName, titleColumn); + } + + @Override + public StringExpression getURL(ColumnInfo parent) + { + return getURL(parent, true); + } + } + + @Override + public @NotNull QueryView createView(ViewContext context, @NotNull QuerySettings settings, BindException errors) + { + if (TableType.DataClasses.name().equalsIgnoreCase(settings.getQueryName())) + { + return new QueryView(this, settings, errors) + { + @Override + protected boolean canInsert() + { + TableInfo table = getTable(); + return table != null && table.hasPermission(getUser(), InsertPermission.class); + } + + @Override + public boolean showImportDataButton() + { + return false; + } + }; + } + + QueryView queryView = super.createView(context, settings, errors); + + if (TableType.Materials.name().equalsIgnoreCase(settings.getQueryName()) || + TableType.Data.name().equalsIgnoreCase(settings.getQueryName()) || + TableType.Protocols.name().equalsIgnoreCase(settings.getQueryName())) + { + // Use default delete button, but without showing the confirmation text + queryView.setShowDeleteButtonConfirmationText(false); + } + + return queryView; + } + + public enum DerivationDataScopeType + { + ChildOnly, + ParentOnly, + All + } +} diff --git a/api/src/org/labkey/api/exp/query/ExpStaleSampleFilesTable.java b/api/src/org/labkey/api/exp/query/ExpStaleSampleFilesTable.java new file mode 100644 index 00000000000..67f7dfd9eb9 --- /dev/null +++ b/api/src/org/labkey/api/exp/query/ExpStaleSampleFilesTable.java @@ -0,0 +1,101 @@ +package org.labkey.api.exp.query; + +import org.jetbrains.annotations.NotNull; +import org.labkey.api.data.BaseColumnInfo; +import org.labkey.api.data.ContainerFilter; +import org.labkey.api.data.CoreSchema; +import org.labkey.api.data.JdbcType; +import org.labkey.api.data.SQLFragment; +import org.labkey.api.data.TableInfo; +import org.labkey.api.data.VirtualTable; +import org.labkey.api.exp.api.ExperimentService; +import org.labkey.api.files.FileContentService; +import org.labkey.api.query.FilteredTable; +import org.labkey.api.query.UserIdForeignKey; +import org.labkey.api.query.UserSchema; +import org.labkey.api.query.column.BuiltInColumnTypes; + +public class ExpStaleSampleFilesTable extends FilteredTable +{ + public ExpStaleSampleFilesTable(@NotNull ExpSchema schema, ContainerFilter cf) + { + super(createVirtualTable(schema), schema, cf); + wrapAllColumns(true); + } + + private static TableInfo createVirtualTable(@NotNull ExpSchema schema) + { + return new ExpStaleSampleFilesTable.FileUnionTable(schema); + } + + private static class FileUnionTable extends VirtualTable + { + private final SQLFragment _query; + + public FileUnionTable(@NotNull UserSchema schema) + { + super(CoreSchema.getInstance().getSchema(), ExpSchema.SAMPLE_FILES_TABLE, schema); + + FileContentService svc = FileContentService.get(); + _query = new SQLFragment(); + _query.appendComment("", getSchema().getSqlDialect()); + + TableInfo expDataTable = ExperimentService.get().getTinfoData(); + TableInfo materialTable = ExperimentService.get().getTinfoMaterial(); + + SQLFragment sampleFileSql = new SQLFragment("SELECT m.Container, if.FilePathShort \n") + .append("FROM (") + .append(svc.listSampleFilesQuery(schema.getUser())) + .append(") AS if \n") + .append("JOIN ") + .append(materialTable, "m") + .append(" ON if.SourceKey = m.RowId"); + + SQLFragment staleFileSql = new SQLFragment("SELECT ed.name as filename, ed.container, ed.created, ed.createdBy, ed.DataFileUrl FROM ") + .append(expDataTable, "ed") + .append(" LEFT JOIN (") + .append(sampleFileSql) + .append(" ) sf\n") + .append(" ON ed.name = sf.FilePathShort AND ed.container = sf.container\n") + .append(" WHERE ed.datafileurl LIKE ") + .appendValue("%@files/sampletype/%") + .append(" AND sf.FilePathShort IS NULL"); + + _query.append(staleFileSql); + + _query.appendComment("", getSchema().getSqlDialect()); + + + var filePathShortCol = new BaseColumnInfo("FileName", this, JdbcType.VARCHAR); + addColumn(filePathShortCol); + + if (schema.getUser().hasApplicationAdminPermission()) + { + var filePathCol = new BaseColumnInfo("DataFileUrl", this, JdbcType.VARCHAR); + filePathCol.setHidden(true); + addColumn(filePathCol); + } + + var containerCol = new BaseColumnInfo("Container", this, JdbcType.VARCHAR); + containerCol.setConceptURI(BuiltInColumnTypes.CONTAINERID_CONCEPT_URI); + addColumn(containerCol); + + var createdCol = new BaseColumnInfo("Created", this, JdbcType.DATE); + addColumn(createdCol); + + var createdByCol = new BaseColumnInfo("CreatedBy", this, JdbcType.INTEGER); + UserIdForeignKey.initColumn(createdByCol); + addColumn(createdByCol); + + } + + @NotNull + @Override + public SQLFragment getFromSQL() + { + return _query; + } + + } + +} diff --git a/api/src/org/labkey/api/files/FileContentService.java b/api/src/org/labkey/api/files/FileContentService.java index 9791ba68902..9ac57fb9598 100644 --- a/api/src/org/labkey/api/files/FileContentService.java +++ b/api/src/org/labkey/api/files/FileContentService.java @@ -1,360 +1,362 @@ -/* - * Copyright (c) 2009-2019 LabKey Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.labkey.api.files; - -import org.jetbrains.annotations.NotNull; -import org.jetbrains.annotations.Nullable; -import org.labkey.api.attachments.AttachmentDirectory; -import org.labkey.api.data.Container; -import org.labkey.api.data.SQLFragment; -import org.labkey.api.data.TableInfo; -import org.labkey.api.exp.api.DataType; -import org.labkey.api.exp.api.ExpData; -import org.labkey.api.exp.query.ExpDataTable; -import org.labkey.api.query.QueryUpdateService; -import org.labkey.api.security.User; -import org.labkey.api.services.ServiceRegistry; -import org.labkey.api.util.FileUtil; -import org.labkey.api.webdav.WebdavResource; - -import java.io.File; -import java.net.URI; -import java.nio.file.Path; -import java.util.Collection; -import java.util.List; -import java.util.Map; - -/** - * User: klum - * Date: Dec 9, 2009 - */ -public interface FileContentService -{ - String UPLOADED_FILE_NAMESPACE_PREFIX = "UploadedFile"; - DataType UPLOADED_FILE = new DataType(UPLOADED_FILE_NAMESPACE_PREFIX); - - String FILES_LINK = "@files"; - String FILE_SETS_LINK = "@filesets"; - String PIPELINE_LINK = "@pipeline"; - String SCRIPTS_LINK = "@scripts"; - String CLOUD_LINK = "@cloud"; - String ASSAY_FILES = "@assayfiles"; - - String CLOUD_ROOT_PREFIX = "/@cloud"; - - static @Nullable FileContentService get() - { - return ServiceRegistry.get().getService(FileContentService.class); - } - - static void setInstance(FileContentService impl) - { - ServiceRegistry.get().registerService(FileContentService.class, impl); - } - - /** - * Returns a list of Container in which the path resides. - */ - @NotNull - List getContainersForFilePath(java.nio.file.Path path); - - /** - * Returns the file root of the specified container. If not explicitly defined, - * it will default to a path relative to the first parent container with an override - */ - @Nullable - File getFileRoot(@NotNull Container c); - - @Nullable - java.nio.file.Path getFileRootPath(@NotNull Container c); - - /** - * Returns the file root of the specified content type for a container - */ - @Nullable - File getFileRoot(@NotNull Container c, @NotNull ContentType type); - - @Nullable - java.nio.file.Path getFileRootPath(@NotNull Container c, @NotNull ContentType type); - - @Nullable - URI getFileRootUri(@NotNull Container c, @NotNull ContentType type, @Nullable String filePath); - - void setFileRoot(@NotNull Container c, @Nullable File root); - - void setFileRootPath(@NotNull Container c, @Nullable String root); - - void setCloudRoot(@NotNull Container c, String cloudRootName); - - boolean isCloudRoot(Container container); - - String getCloudRootName(Container c); - - void disableFileRoot(Container container); - - boolean isFileRootDisabled(Container container); - - /** - * A file root can use a default root based on a single site wide root that mirrors the folder structure of - * a project. - */ - boolean isUseDefaultRoot(Container container); - - void setIsUseDefaultRoot(Container container, boolean useDefaultRoot); - - - @NotNull - File getSiteDefaultRoot(); - - @NotNull - Path getSiteDefaultRootPath(); - - @Nullable - String getProblematicFileRootMessage(); - - void setSiteDefaultRoot(File root, User user); - - void setFileRootSetViaStartupProperty(boolean fileRootSetViaStartupProperty); - - boolean isFileRootSetViaStartupProperty(); - - /** - * Create an attachmentParent object that will allow storing files in the file system - * - * @param c Container this will be attached to - * @param name Name of the parent used in getMappedAttachmentDirectory - * @param path Path to the file. If relative is true, this is the name of a subdirectory of the directory mapped to this c - * container. If relative is false, this is a fully qualified path name - * @param relative if true, path is a relative path from the directory mapped from the container - * @return the created attachment parent - */ - AttachmentDirectory registerDirectory(Container c, String name, String path, boolean relative); - - /** - * Forget about a named directory - * - * @param c Container for this attachmentParent - * @param label Name of the parent used in registerDirectory - */ - void unregisterDirectory(Container c, String label); - - /** - * Return an AttachmentParent for files in the directory mapped to this container - * - * @param c Container in the file system - * @param createDir Create the mapped directory if it doesn't exist - * @return AttachmentParent that can be passed to other methods of this interface - */ - @Nullable - AttachmentDirectory getMappedAttachmentDirectory(Container c, boolean createDir) throws UnsetRootDirectoryException, MissingRootDirectoryException; - - @Nullable - AttachmentDirectory getMappedAttachmentDirectory(Container c, ContentType contentType, boolean createDir) throws UnsetRootDirectoryException, MissingRootDirectoryException; - - /** - * Return a named AttachmentParent for files in the directory mapped to this container - * - * @param c Container in the file system - * @return AttachmentParent that can be passed to other methods of this interface - */ - AttachmentDirectory getRegisteredDirectory(Container c, String label); - - /** - * Return a named AttachmentParent for files in the directory mapped to this container - * - * @param c Container in the file system - * @return AttachmentParent that can be passed to other methods of this interface - */ - AttachmentDirectory getRegisteredDirectoryFromEntityId(Container c, String entityId); - - /** - * Return true if the supplied string is a valid project root - * - * @param root String to use as the file path - * @return boolean - */ - boolean isValidProjectRoot(String root); - - /** - * Return all AttachmentParents for files in the directory mapped to this container - * - * @param c Container in the file system - * @return Collection of attachment directories that have previously been registered - */ - @NotNull Collection getRegisteredDirectories(Container c); - - enum ContentType { - files, - pipeline, - assay, - scripts, - assayfiles - } - - String getFolderName(ContentType type); - - FilesAdminOptions getAdminOptions(Container c); - - void setAdminOptions(Container c, FilesAdminOptions options); - - void setAdminOptions(Container c, String properties); - - /** - * Returns the default file root of the specified container. This will default to a path - * relative to the first parent container with an override - */ - File getDefaultRoot(Container c, boolean createDir); - Path getDefaultRootPath(@NotNull Container c, boolean createDir); - - class DefaultRootInfo - { - private final java.nio.file.Path _path; - private final String _prettyStr; - private final boolean _isCloud; - private final String _cloudName; - - public DefaultRootInfo(java.nio.file.Path path, String prettyStr, boolean isCloud, String cloudName) - { - _path = path; - _prettyStr = prettyStr; - _isCloud = isCloud; - _cloudName = cloudName; - } - - public java.nio.file.Path getPath() - { - return _path; - } - - public String getPrettyStr() - { - return _prettyStr; - } - - public boolean isCloud() - { - return _isCloud; - } - - public String getCloudName() - { - return _cloudName; - } - } - - DefaultRootInfo getDefaultRootInfo(Container container); - - String getDomainURI(Container c); - - String getDomainURI(Container c, FilesAdminOptions.fileConfig config); - - ExpData getDataObject(WebdavResource resource, Container c); - QueryUpdateService getFilePropsUpdateService(TableInfo tinfo, Container container); - - void moveFileRoot(File prev, File dest, @Nullable User user, @Nullable Container container); - default void moveFileRoot(Path prev, Path dest, @Nullable User user, @Nullable Container container) - { - if (!FileUtil.hasCloudScheme(prev) && !FileUtil.hasCloudScheme(dest)) - { - moveFileRoot(prev.toFile(), dest.toFile(), user, container); - } - } - - /** Notifies all registered FileListeners that a file or directory has been created */ - void fireFileCreateEvent(@NotNull File created, @Nullable User user, @Nullable Container container); - default void fireFileCreateEvent(@NotNull Path created, @Nullable User user, @Nullable Container container) - { - if (!FileUtil.hasCloudScheme(created)) - fireFileCreateEvent(created.toFile(), user, container); - } - /** - * Notifies all registered FileListeners that a file or directory has moved - * @return number of rows updated across all listeners - */ - int fireFileMoveEvent(@NotNull File src, @NotNull File dest, @Nullable User user, @Nullable Container container); - default int fireFileMoveEvent(@NotNull Path src, @NotNull Path dest, @Nullable User user, @Nullable Container container) - { - if (!FileUtil.hasCloudScheme(src) && !FileUtil.hasCloudScheme(dest)) - return fireFileMoveEvent(src.toFile(), dest.toFile(), user, container); - return 0; - } - default int fireFileMoveEvent(@NotNull Path src, @NotNull Path dest, @Nullable User user, @Nullable Container sourceContainer, @Nullable Container targetContainer) - { - return fireFileMoveEvent(src, dest, user, sourceContainer); - } - - /** Notifies all registered FileListeners that a file or directory has been replaced */ - default void fireFileReplacedEvent(@NotNull Path replaced, @Nullable User user, @Nullable Container container){} - - /** Notifies all registered FileListeners that a file or directory has been deleted */ - default void fireFileDeletedEvent(@NotNull Path deleted, @Nullable User user, @Nullable Container container){} - - /** Add a listener that will be notified when files are created or are moved */ - void addFileListener(FileListener listener); - - Map> listFiles(@NotNull Container container); - - /** - * Returns a SQLFragment for file paths that this FileListener is aware of when the user is a site admin, or empty - * results otherwise. - * The expected columns are: - *
    - *
  • Container
    • - *
  • Created
    • - *
  • CreatedBy
    • - *
  • Modified
    • - *
  • ModifiedBy
    • - *
  • FilePath
    • - *
  • SourceKey
    • - *
  • SourceName
    • - *
- */ - SQLFragment listFilesQuery(@NotNull User currentUser); - - void setWebfilesEnabled(boolean enabled, User user); - - /** - * Return file's virtual folder path that's relative to container's file root. Roots are matched in order of @assayfiles, @files, @pipeline and then each @filesets. - * @param dataFileUrl The data file Url of file - * @param container Container in the file system - * @return folder relative to file root - */ - String getDataFileRelativeFileRootPath(@NotNull String dataFileUrl, Container container); - - enum PathType { full, serverRelative, folderRelative } - - @Nullable - URI getWebDavUrl(@NotNull Path path, @NotNull Container container, @NotNull PathType type); - - /** - * Ensure an entry in the exp.data table exists for all files in the container's file root. - */ - void ensureFileData(@NotNull ExpDataTable table); - - /** - * Allows a module to register a directory pattern to be checked in the files webpart in order to zip the matching directory before uploading. - * @param directoryPattern DirectoryPattern - * */ - void addZiploaderPattern(DirectoryPattern directoryPattern); - - /** - * Returns a list of DirectoryPattern objects for the active modules in the given container. - * */ - List getZiploaderPatterns(Container container); - - File getMoveTargetFile(String absoluteFilePath, @NotNull Container sourceContainer, @NotNull Container targetContainer); -} +/* + * Copyright (c) 2009-2019 LabKey Corporation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.labkey.api.files; + +import org.jetbrains.annotations.NotNull; +import org.jetbrains.annotations.Nullable; +import org.labkey.api.attachments.AttachmentDirectory; +import org.labkey.api.data.Container; +import org.labkey.api.data.SQLFragment; +import org.labkey.api.data.TableInfo; +import org.labkey.api.exp.api.DataType; +import org.labkey.api.exp.api.ExpData; +import org.labkey.api.exp.query.ExpDataTable; +import org.labkey.api.query.QueryUpdateService; +import org.labkey.api.security.User; +import org.labkey.api.services.ServiceRegistry; +import org.labkey.api.util.FileUtil; +import org.labkey.api.webdav.WebdavResource; + +import java.io.File; +import java.net.URI; +import java.nio.file.Path; +import java.util.Collection; +import java.util.List; +import java.util.Map; + +/** + * User: klum + * Date: Dec 9, 2009 + */ +public interface FileContentService +{ + String UPLOADED_FILE_NAMESPACE_PREFIX = "UploadedFile"; + DataType UPLOADED_FILE = new DataType(UPLOADED_FILE_NAMESPACE_PREFIX); + + String FILES_LINK = "@files"; + String FILE_SETS_LINK = "@filesets"; + String PIPELINE_LINK = "@pipeline"; + String SCRIPTS_LINK = "@scripts"; + String CLOUD_LINK = "@cloud"; + String ASSAY_FILES = "@assayfiles"; + + String CLOUD_ROOT_PREFIX = "/@cloud"; + + static @Nullable FileContentService get() + { + return ServiceRegistry.get().getService(FileContentService.class); + } + + static void setInstance(FileContentService impl) + { + ServiceRegistry.get().registerService(FileContentService.class, impl); + } + + /** + * Returns a list of Container in which the path resides. + */ + @NotNull + List getContainersForFilePath(java.nio.file.Path path); + + /** + * Returns the file root of the specified container. If not explicitly defined, + * it will default to a path relative to the first parent container with an override + */ + @Nullable + File getFileRoot(@NotNull Container c); + + @Nullable + java.nio.file.Path getFileRootPath(@NotNull Container c); + + /** + * Returns the file root of the specified content type for a container + */ + @Nullable + File getFileRoot(@NotNull Container c, @NotNull ContentType type); + + @Nullable + java.nio.file.Path getFileRootPath(@NotNull Container c, @NotNull ContentType type); + + @Nullable + URI getFileRootUri(@NotNull Container c, @NotNull ContentType type, @Nullable String filePath); + + void setFileRoot(@NotNull Container c, @Nullable File root); + + void setFileRootPath(@NotNull Container c, @Nullable String root); + + void setCloudRoot(@NotNull Container c, String cloudRootName); + + boolean isCloudRoot(Container container); + + String getCloudRootName(Container c); + + void disableFileRoot(Container container); + + boolean isFileRootDisabled(Container container); + + /** + * A file root can use a default root based on a single site wide root that mirrors the folder structure of + * a project. + */ + boolean isUseDefaultRoot(Container container); + + void setIsUseDefaultRoot(Container container, boolean useDefaultRoot); + + + @NotNull + File getSiteDefaultRoot(); + + @NotNull + Path getSiteDefaultRootPath(); + + @Nullable + String getProblematicFileRootMessage(); + + void setSiteDefaultRoot(File root, User user); + + void setFileRootSetViaStartupProperty(boolean fileRootSetViaStartupProperty); + + boolean isFileRootSetViaStartupProperty(); + + /** + * Create an attachmentParent object that will allow storing files in the file system + * + * @param c Container this will be attached to + * @param name Name of the parent used in getMappedAttachmentDirectory + * @param path Path to the file. If relative is true, this is the name of a subdirectory of the directory mapped to this c + * container. If relative is false, this is a fully qualified path name + * @param relative if true, path is a relative path from the directory mapped from the container + * @return the created attachment parent + */ + AttachmentDirectory registerDirectory(Container c, String name, String path, boolean relative); + + /** + * Forget about a named directory + * + * @param c Container for this attachmentParent + * @param label Name of the parent used in registerDirectory + */ + void unregisterDirectory(Container c, String label); + + /** + * Return an AttachmentParent for files in the directory mapped to this container + * + * @param c Container in the file system + * @param createDir Create the mapped directory if it doesn't exist + * @return AttachmentParent that can be passed to other methods of this interface + */ + @Nullable + AttachmentDirectory getMappedAttachmentDirectory(Container c, boolean createDir) throws UnsetRootDirectoryException, MissingRootDirectoryException; + + @Nullable + AttachmentDirectory getMappedAttachmentDirectory(Container c, ContentType contentType, boolean createDir) throws UnsetRootDirectoryException, MissingRootDirectoryException; + + /** + * Return a named AttachmentParent for files in the directory mapped to this container + * + * @param c Container in the file system + * @return AttachmentParent that can be passed to other methods of this interface + */ + AttachmentDirectory getRegisteredDirectory(Container c, String label); + + /** + * Return a named AttachmentParent for files in the directory mapped to this container + * + * @param c Container in the file system + * @return AttachmentParent that can be passed to other methods of this interface + */ + AttachmentDirectory getRegisteredDirectoryFromEntityId(Container c, String entityId); + + /** + * Return true if the supplied string is a valid project root + * + * @param root String to use as the file path + * @return boolean + */ + boolean isValidProjectRoot(String root); + + /** + * Return all AttachmentParents for files in the directory mapped to this container + * + * @param c Container in the file system + * @return Collection of attachment directories that have previously been registered + */ + @NotNull Collection getRegisteredDirectories(Container c); + + enum ContentType { + files, + pipeline, + assay, + scripts, + assayfiles + } + + String getFolderName(ContentType type); + + FilesAdminOptions getAdminOptions(Container c); + + void setAdminOptions(Container c, FilesAdminOptions options); + + void setAdminOptions(Container c, String properties); + + /** + * Returns the default file root of the specified container. This will default to a path + * relative to the first parent container with an override + */ + File getDefaultRoot(Container c, boolean createDir); + Path getDefaultRootPath(@NotNull Container c, boolean createDir); + + class DefaultRootInfo + { + private final java.nio.file.Path _path; + private final String _prettyStr; + private final boolean _isCloud; + private final String _cloudName; + + public DefaultRootInfo(java.nio.file.Path path, String prettyStr, boolean isCloud, String cloudName) + { + _path = path; + _prettyStr = prettyStr; + _isCloud = isCloud; + _cloudName = cloudName; + } + + public java.nio.file.Path getPath() + { + return _path; + } + + public String getPrettyStr() + { + return _prettyStr; + } + + public boolean isCloud() + { + return _isCloud; + } + + public String getCloudName() + { + return _cloudName; + } + } + + DefaultRootInfo getDefaultRootInfo(Container container); + + String getDomainURI(Container c); + + String getDomainURI(Container c, FilesAdminOptions.fileConfig config); + + ExpData getDataObject(WebdavResource resource, Container c); + QueryUpdateService getFilePropsUpdateService(TableInfo tinfo, Container container); + + void moveFileRoot(File prev, File dest, @Nullable User user, @Nullable Container container); + default void moveFileRoot(Path prev, Path dest, @Nullable User user, @Nullable Container container) + { + if (!FileUtil.hasCloudScheme(prev) && !FileUtil.hasCloudScheme(dest)) + { + moveFileRoot(prev.toFile(), dest.toFile(), user, container); + } + } + + /** Notifies all registered FileListeners that a file or directory has been created */ + void fireFileCreateEvent(@NotNull File created, @Nullable User user, @Nullable Container container); + default void fireFileCreateEvent(@NotNull Path created, @Nullable User user, @Nullable Container container) + { + if (!FileUtil.hasCloudScheme(created)) + fireFileCreateEvent(created.toFile(), user, container); + } + /** + * Notifies all registered FileListeners that a file or directory has moved + * @return number of rows updated across all listeners + */ + int fireFileMoveEvent(@NotNull File src, @NotNull File dest, @Nullable User user, @Nullable Container container); + default int fireFileMoveEvent(@NotNull Path src, @NotNull Path dest, @Nullable User user, @Nullable Container container) + { + if (!FileUtil.hasCloudScheme(src) && !FileUtil.hasCloudScheme(dest)) + return fireFileMoveEvent(src.toFile(), dest.toFile(), user, container); + return 0; + } + default int fireFileMoveEvent(@NotNull Path src, @NotNull Path dest, @Nullable User user, @Nullable Container sourceContainer, @Nullable Container targetContainer) + { + return fireFileMoveEvent(src, dest, user, sourceContainer); + } + + /** Notifies all registered FileListeners that a file or directory has been replaced */ + default void fireFileReplacedEvent(@NotNull Path replaced, @Nullable User user, @Nullable Container container){} + + /** Notifies all registered FileListeners that a file or directory has been deleted */ + default void fireFileDeletedEvent(@NotNull Path deleted, @Nullable User user, @Nullable Container container){} + + /** Add a listener that will be notified when files are created or are moved */ + void addFileListener(FileListener listener); + + Map> listFiles(@NotNull Container container); + + /** + * Returns a SQLFragment for file paths that this FileListener is aware of when the user is a site admin, or empty + * results otherwise. + * The expected columns are: + *
    + *
  • Container
    • + *
  • Created
    • + *
  • CreatedBy
    • + *
  • Modified
    • + *
  • ModifiedBy
    • + *
  • FilePath
    • + *
  • SourceKey
    • + *
  • SourceName
    • + *
+ */ + SQLFragment listFilesQuery(@NotNull User currentUser); + + SQLFragment listSampleFilesQuery(@NotNull User currentUser); + + void setWebfilesEnabled(boolean enabled, User user); + + /** + * Return file's virtual folder path that's relative to container's file root. Roots are matched in order of @assayfiles, @files, @pipeline and then each @filesets. + * @param dataFileUrl The data file Url of file + * @param container Container in the file system + * @return folder relative to file root + */ + String getDataFileRelativeFileRootPath(@NotNull String dataFileUrl, Container container); + + enum PathType { full, serverRelative, folderRelative } + + @Nullable + URI getWebDavUrl(@NotNull Path path, @NotNull Container container, @NotNull PathType type); + + /** + * Ensure an entry in the exp.data table exists for all files in the container's file root. + */ + void ensureFileData(@NotNull ExpDataTable table); + + /** + * Allows a module to register a directory pattern to be checked in the files webpart in order to zip the matching directory before uploading. + * @param directoryPattern DirectoryPattern + * */ + void addZiploaderPattern(DirectoryPattern directoryPattern); + + /** + * Returns a list of DirectoryPattern objects for the active modules in the given container. + * */ + List getZiploaderPatterns(Container container); + + File getMoveTargetFile(String absoluteFilePath, @NotNull Container sourceContainer, @NotNull Container targetContainer); +} diff --git a/api/src/org/labkey/api/files/FileListener.java b/api/src/org/labkey/api/files/FileListener.java index 858d1f54306..218aac7e098 100644 --- a/api/src/org/labkey/api/files/FileListener.java +++ b/api/src/org/labkey/api/files/FileListener.java @@ -1,97 +1,102 @@ -/* - * Copyright (c) 2013-2019 LabKey Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.labkey.api.files; - -import org.jetbrains.annotations.NotNull; -import org.jetbrains.annotations.Nullable; -import org.labkey.api.data.Container; -import org.labkey.api.data.SQLFragment; -import org.labkey.api.security.User; -import org.labkey.api.util.FileUtil; - -import java.io.File; -import java.nio.file.Path; -import java.util.Collection; - -/** - * Listener that gets notified when the server moves files or directories on the file system. Method is invoked - * once at the root level of the move, not recursively for every child file. - * User: jeckels - * Date: 11/7/12 - */ -public interface FileListener -{ - String getSourceName(); - - /** - * Called AFTER the file (or directory) has already been created on disk - * @param created newly created resource - * @param user if available, the user who initiated the create - * @param container if available, the container in which the create was initiated - */ - void fileCreated(@NotNull File created, @Nullable User user, @Nullable Container container); - default void fileCreated(@NotNull Path created, @Nullable User user, @Nullable Container container) - { - if (!FileUtil.hasCloudScheme(created)) - fileCreated(created.toFile(), user, container); - } - - /** - * Called AFTER the file (or directory) has already been moved on disk - * @param src the original file path - * @param dest the new file path - * @param user if available, the user who initiated the move - * @param container if available, the container in which the move was initiated - */ - int fileMoved(@NotNull File src, @NotNull File dest, @Nullable User user, @Nullable Container container); - default int fileMoved(@NotNull Path src, @NotNull Path dest, @Nullable User user, @Nullable Container container) - { - if (!FileUtil.hasCloudScheme(src) && !FileUtil.hasCloudScheme(dest)) - return fileMoved(src.toFile(), dest.toFile(), user, container); - return 0; - } - default int fileMoved(@NotNull Path src, @NotNull Path dest, @Nullable User user, @Nullable Container sourceContainer, @Nullable Container targetContainer) - { - return fileMoved(src, dest, user, sourceContainer); - } - - default void fileReplaced(@NotNull Path replaced, @Nullable User user, @Nullable Container container){} - - default void fileDeleted(@NotNull Path deleted, @Nullable User user, @Nullable Container container) {} - - /** - * List file paths in the database this FileListener is aware of. - * @param container If not null, list files in the given container, otherwise from all containers. - */ - Collection listFiles(@Nullable Container container); // Nobody really calls this -// public Collection listFilePaths(@Nullable Container container); - - /** - * Returns a SQLFragment for file paths that this FileListener is aware of. - * The expected columns are: - *
    - *
  • Container
    • - *
  • Created
    • - *
  • CreatedBy
    • - *
  • Modified
    • - *
  • ModifiedBy
    • - *
  • FilePath
    • - *
  • SourceKey
    • - *
  • SourceName
    • - *
- */ - SQLFragment listFilesQuery(); -} +/* + * Copyright (c) 2013-2019 LabKey Corporation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.labkey.api.files; + +import org.jetbrains.annotations.NotNull; +import org.jetbrains.annotations.Nullable; +import org.labkey.api.data.Container; +import org.labkey.api.data.SQLFragment; +import org.labkey.api.security.User; +import org.labkey.api.util.FileUtil; + +import java.io.File; +import java.nio.file.Path; +import java.util.Collection; + +/** + * Listener that gets notified when the server moves files or directories on the file system. Method is invoked + * once at the root level of the move, not recursively for every child file. + * User: jeckels + * Date: 11/7/12 + */ +public interface FileListener +{ + String getSourceName(); + + /** + * Called AFTER the file (or directory) has already been created on disk + * @param created newly created resource + * @param user if available, the user who initiated the create + * @param container if available, the container in which the create was initiated + */ + void fileCreated(@NotNull File created, @Nullable User user, @Nullable Container container); + default void fileCreated(@NotNull Path created, @Nullable User user, @Nullable Container container) + { + if (!FileUtil.hasCloudScheme(created)) + fileCreated(created.toFile(), user, container); + } + + /** + * Called AFTER the file (or directory) has already been moved on disk + * @param src the original file path + * @param dest the new file path + * @param user if available, the user who initiated the move + * @param container if available, the container in which the move was initiated + */ + int fileMoved(@NotNull File src, @NotNull File dest, @Nullable User user, @Nullable Container container); + default int fileMoved(@NotNull Path src, @NotNull Path dest, @Nullable User user, @Nullable Container container) + { + if (!FileUtil.hasCloudScheme(src) && !FileUtil.hasCloudScheme(dest)) + return fileMoved(src.toFile(), dest.toFile(), user, container); + return 0; + } + default int fileMoved(@NotNull Path src, @NotNull Path dest, @Nullable User user, @Nullable Container sourceContainer, @Nullable Container targetContainer) + { + return fileMoved(src, dest, user, sourceContainer); + } + + default void fileReplaced(@NotNull Path replaced, @Nullable User user, @Nullable Container container){} + + default void fileDeleted(@NotNull Path deleted, @Nullable User user, @Nullable Container container) {} + + /** + * List file paths in the database this FileListener is aware of. + * @param container If not null, list files in the given container, otherwise from all containers. + */ + Collection listFiles(@Nullable Container container); // Nobody really calls this +// public Collection listFilePaths(@Nullable Container container); + + /** + * Returns a SQLFragment for file paths that this FileListener is aware of. + * The expected columns are: + *
    + *
  • Container
    • + *
  • Created
    • + *
  • CreatedBy
    • + *
  • Modified
    • + *
  • ModifiedBy
    • + *
  • FilePath
    • + *
  • SourceKey
    • + *
  • SourceName
    • + *
+ */ + SQLFragment listFilesQuery(); + + default SQLFragment listSampleFilesQuery() + { + return null; + } +} diff --git a/api/src/org/labkey/api/files/TableUpdaterFileListener.java b/api/src/org/labkey/api/files/TableUpdaterFileListener.java index ce46e143c87..2b276896bda 100644 --- a/api/src/org/labkey/api/files/TableUpdaterFileListener.java +++ b/api/src/org/labkey/api/files/TableUpdaterFileListener.java @@ -1,451 +1,461 @@ -/* - * Copyright (c) 2013-2019 LabKey Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.labkey.api.files; - -import org.apache.commons.lang3.StringUtils; -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; -import org.jetbrains.annotations.NotNull; -import org.jetbrains.annotations.Nullable; -import org.labkey.api.data.ColumnInfo; -import org.labkey.api.data.CompareType; -import org.labkey.api.data.Container; -import org.labkey.api.data.DbSchema; -import org.labkey.api.data.SQLFragment; -import org.labkey.api.data.SimpleFilter; -import org.labkey.api.data.Sort; -import org.labkey.api.data.SqlExecutor; -import org.labkey.api.data.Table; -import org.labkey.api.data.TableInfo; -import org.labkey.api.data.TableSelector; -import org.labkey.api.data.dialect.SqlDialect; -import org.labkey.api.security.User; -import org.labkey.api.util.FileUtil; - -import java.io.File; -import java.nio.file.Files; -import java.nio.file.Path; -import java.util.Collection; -import java.util.Collections; -import java.util.Date; -import java.util.Set; - -/** - * FileListener implementation that can update tables that store file paths in various flavors (URI, standard OS - * paths, etc). - * User: jeckels - * Date: 11/7/12 - */ -public class TableUpdaterFileListener implements FileListener -{ - private static final Logger LOG = LogManager.getLogger(TableUpdaterFileListener.class); - - public static final String TABLE_ALIAS = "x"; - - private final TableInfo _table; - private final SQLFragment _containerFrag; - private final ColumnInfo _pathColumn; - private final PathGetter _pathGetter; - private final ColumnInfo _keyColumn; - - public interface PathGetter - { - /** @return the string that is expected to be in the database */ - String get(File f); - String get(Path f); - /** @return the file path separator (typically '/' or '\') */ - String getSeparatorSuffix(Path path); - } - - public enum Type implements PathGetter - { - /** Turns the file path into a "file:"-prefixed URI */ - uri - { - @Override - public String get(Path path) - { - return FileUtil.pathToString(path); - } - - @Override - public String get(File f) - { - return FileUtil.uriToString(f.toURI()); - } - - @Override - public String getSeparatorSuffix(Path path) - { - return "/"; - } - }, - - /** Just uses getPath() to turn the File into a String */ - filePath - { - @Override - public String get(Path path) - { - if (FileUtil.hasCloudScheme(path)) - return FileUtil.pathToString(path); - else - return get(path.toFile()); - } - - @Override - public String get(File f) - { - return f.getPath(); - } - - @Override - public String getSeparatorSuffix(Path path) - { - return FileUtil.hasCloudScheme(path) ? "/" : File.separator; - } - }, - - /** field has root of file path */ - fileRootPath - { - @Override - public String get(Path path) - { - return filePath.get(path); - } - - @Override - public String get(File f) - { - return filePath.get(f); - } - - @Override - public String getSeparatorSuffix(Path path) - { - return FileUtil.hasCloudScheme(path) ? "/" : File.separator; - } - }, - - /** Just uses getPath() to turn the File into a String, but replaces all backslashes with forward slashes */ - filePathForwardSlash - { - @Override - public String get(Path path) - { - if (FileUtil.hasCloudScheme(path)) - return FileUtil.pathToString(path); - else - return get(path.toFile()); - } - - @Override - public String get(File f) - { - return f.getPath().replace('\\', '/'); - } - - @Override - public String getSeparatorSuffix(Path path) - { - return "/"; - } - } - } - - public TableUpdaterFileListener(TableInfo table, String pathColumn, PathGetter pathGetter) - { - this(table, pathColumn, pathGetter, null, null); - } - - public TableUpdaterFileListener(TableInfo table, String pathColumn, PathGetter pathGetter, @Nullable String keyColumn) - { - this(table, pathColumn, pathGetter, keyColumn, null); - } - - public TableUpdaterFileListener(TableInfo table, String pathColumnName, PathGetter pathGetter, @Nullable String keyColumnName, @Nullable SQLFragment containerFrag) - { - _table = table; - _pathColumn = table.getColumn(pathColumnName); - if (null == _pathColumn) - throw new IllegalStateException("Column not found: " + pathColumnName); - _keyColumn = null == keyColumnName ? null : table.getColumn(keyColumnName); - if (null != keyColumnName && null == _keyColumn) - throw new IllegalStateException("Column not found: " + keyColumnName); - _pathGetter = pathGetter; - _containerFrag = containerFrag; - } - - @Override - public String getSourceName() - { - return _table.getSchema().getName() + "." + _table.getName() + "." + _pathColumn.getName(); - } - - public String getSourceSelect(SqlDialect sqlDialect) - { - String schema = sqlDialect.getStringHandler().quoteStringLiteral(_table.getSchema().getName()); - String table = sqlDialect.getStringHandler().quoteStringLiteral(_table.getName()); - String column = sqlDialect.getStringHandler().quoteStringLiteral(_pathColumn.getName()); - return schema + "." + table + "." + column; - } - - @Override - public void fileCreated(@NotNull File created, @Nullable User user, @Nullable Container container) - { - } - - @Override - public void fileCreated(@NotNull Path created, @Nullable User user, @Nullable Container container) - { - } - - @Override - public int fileMoved(@NotNull File src, @NotNull File dest, @Nullable User user, @Nullable Container container) - { - return fileMoved(src.toPath(), dest.toPath(), user, container); - } - - @Override - public int fileMoved(@NotNull Path src, @NotNull Path dest, @Nullable User user, @Nullable Container container) - { - DbSchema schema = _table.getSchema(); - SqlDialect dialect = schema.getSqlDialect(); - - // Build up SQL that can be used for both the file and any children - SQLFragment sharedSQL = new SQLFragment("UPDATE "); - sharedSQL.append(_table); - sharedSQL.append(_table.getSqlDialect().isSqlServer() ? " WITH (UPDLOCK)" : ""); - sharedSQL.append(" SET "); - if (_table.getColumn("Modified") != null) - { - sharedSQL.append("Modified = ?, "); - sharedSQL.add(new Date()); - } - if (_table.getColumn("ModifiedBy") != null && user != null) - { - sharedSQL.append("ModifiedBy = ?, "); - sharedSQL.add(user.getUserId()); - } - sharedSQL.appendIdentifier(_pathColumn.getSelectIdentifier()); - sharedSQL.append(" = "); - - String srcPath = getSourcePath(src, container); - String destPath = _pathGetter.get(dest); - String srcPathWithout = null; - - // If it's a directory, prep versions with and without a trailing slash. Check the dest because it's already moved by the time this fires - if (srcPath.endsWith("/") || srcPath.endsWith("\\") || Files.isDirectory(dest)) - { - srcPath = getWithSeparator(src, container); - srcPathWithout = getWithoutSeparator(srcPath); - destPath = getWithoutSeparator(destPath); - } - - // Now build up the SQL to handle this specific path - SQLFragment singleEntrySQL = new SQLFragment(sharedSQL); - singleEntrySQL.append("? WHERE ("); - singleEntrySQL.appendIdentifier(_pathColumn.getSelectIdentifier()); - singleEntrySQL.append(" = ?"); - singleEntrySQL.add(destPath); - singleEntrySQL.add(srcPath); - if (null != srcPathWithout) - { - singleEntrySQL.append(" OR "); - singleEntrySQL.appendIdentifier(_pathColumn.getSelectIdentifier()); - singleEntrySQL.append(" = ?"); - singleEntrySQL.add(srcPathWithout); - } - singleEntrySQL.append(")"); - - int rows = schema.getScope().executeWithRetry(tx -> new SqlExecutor(schema).execute(singleEntrySQL)); - LOG.info("Updated " + rows + " row in " + _table + " for move from " + src + " to " + dest); - - // Handle updating child paths, unless we know that the entry is a file. If it's not (either it's a - // directory or it doesn't exist), then try to fix up child records - if ((!Files.exists(dest) || Files.isDirectory(dest))) - { - String separatorSuffix = _pathGetter.getSeparatorSuffix(dest); - if (!srcPath.endsWith(separatorSuffix)) - { - srcPath = srcPath + separatorSuffix; - } - if (!destPath.endsWith(separatorSuffix)) - { - destPath = destPath + separatorSuffix; - } - - int childRowsUpdated = 0; - - // Consider paths with file:///... - if (srcPath.startsWith("file:")) - { - srcPath = "file://" + srcPath.replaceFirst("^file:/+", "/"); - } - SQLFragment whereClause = new SQLFragment(" WHERE "); - whereClause.append(dialect.getStringIndexOfFunction(new SQLFragment("?", srcPath), _pathColumn.getSelectIdentifier().getSql())).append(" = 1"); - - // Make the SQL to handle children - SQLFragment childPathsSQL = new SQLFragment(sharedSQL); - childPathsSQL.append(dialect.concatenate(new SQLFragment("?", destPath), new SQLFragment(dialect.getSubstringFunction(_pathColumn.getSelectIdentifier().getSql(), new SQLFragment(Integer.toString(srcPath.length() + 1)), new SQLFragment("5000"))))); - childPathsSQL.append(whereClause); - childRowsUpdated += new SqlExecutor(schema).execute(childPathsSQL); - - LOG.info("Updated " + childRowsUpdated + " child paths in " + _table + " rows for move from " + src + " to " + dest); - return childRowsUpdated; - } - return 0; - } - - @NotNull - private String getWithSeparator(Path path, Container container) - { - String result = getSourcePath(path, container); - String separator = _pathGetter.getSeparatorSuffix(path); - return result + (result.endsWith(separator) ? "" : separator); - } - - @NotNull - private String getWithoutSeparator(String path) - { - return (path.endsWith("/") || path.endsWith("\\")) ? path.substring(0, path.length() - 1): path; - } - - @Override - public Collection listFiles(@Nullable Container container) - { - Set columns = Collections.singleton(_pathColumn.getName()); - SimpleFilter filter = new SimpleFilter(); - filter.addCondition(_pathColumn, null, CompareType.NONBLANK); - if (container != null) - { - ColumnInfo containerColumn = _table.getColumn("container"); - if (containerColumn == null) - containerColumn = _table.getColumn("folder"); - - if (containerColumn != null) - filter.addCondition(containerColumn, container.getEntityId()); - else - filter.addCondition(new SimpleFilter.SQLClause("1 = 0", null)); - } - - Sort sort = new Sort(_pathColumn.getFieldKey()); - TableSelector selector = new TableSelector(_table, columns, filter, sort); - - selector.setMaxRows(Table.ALL_ROWS); - return selector.getArrayList(File.class); - } - - @Override - public SQLFragment listFilesQuery() - { - return listFilesQuery(false, null); - } - - public SQLFragment listFilesQuery(boolean skipCreatedModified, String filePath) - { - SQLFragment selectFrag = new SQLFragment(); - selectFrag.append("SELECT\n"); - - if (_containerFrag != null) - selectFrag.append("(").append(_containerFrag).append(") AS Container,\n"); - else if (_table.getColumn("Container") != null) - selectFrag.append(" Container,\n"); - else if (_table.getColumn("Folder") != null) - selectFrag.append(" Folder AS Container,\n"); - else - selectFrag.append(" NULL AS Container,\n"); - - if (!skipCreatedModified) - { - if (_table.getColumn("Created") != null) - selectFrag.append(" Created,\n"); - else - selectFrag.append(" NULL AS Created,\n"); - - if (_table.getColumn("CreatedBy") != null) - selectFrag.append(" CreatedBy,\n"); - else - selectFrag.append(" NULL AS CreatedBy,\n"); - - if (_table.getColumn("Modified") != null) - selectFrag.append(" Modified,\n"); - else - selectFrag.append(" NULL AS Modified,\n"); - - if (_table.getColumn("ModifiedBy") != null) - selectFrag.append(" ModifiedBy,\n"); - else - selectFrag.append(" NULL AS ModifiedBy,\n"); - } - - selectFrag.append(" ").appendIdentifier(_pathColumn.getSelectIdentifier()).append(" AS FilePath,\n"); - - if (_keyColumn != null) - selectFrag.append(" ").appendIdentifier(_keyColumn.getSelectIdentifier()).append(" AS SourceKey,\n"); - else - selectFrag.append(" NULL AS SourceKey,\n"); - - //selectFrag.append(" ? AS SourceName\n").add(getName()); - selectFrag.append(" ").appendValue(getSourceSelect(_table.getSchema().getSqlDialect())).append(" AS SourceName\n"); - - selectFrag.append("FROM ").append(_table, TABLE_ALIAS).append("\n"); - selectFrag.append("WHERE ").appendIdentifier(_pathColumn.getSelectIdentifier()); - - if (StringUtils.isEmpty(filePath)) - selectFrag.append(" IS NOT NULL\n"); - else - selectFrag.append(" = ").appendStringLiteral(filePath, _table.getSchema().getSqlDialect()).append("\n"); - - return selectFrag; - } - - @NotNull - private String getSourcePath(Path path, Container container) - { - // For uri pathGetter, check that file path exists in table, looking for legacy as well - if (Type.uri == _pathGetter) - { - String srcPath = _pathGetter.get(path); - if (pathExists(srcPath, container)) - return srcPath; - - if (!FileUtil.hasCloudScheme(path)) - { - srcPath = path.toFile().getPath(); // File path format (/users/...) - if (pathExists(srcPath, container)) - return srcPath; - } - - // use original if none found, for directories - } - - return _pathGetter.get(path); - } - - private boolean pathExists(String srcPath, Container container) - { - - SimpleFilter filter = (null != _table.getColumn("container")) ? - SimpleFilter.createContainerFilter(container) : - (null != _table.getColumn("folder")) ? - SimpleFilter.createContainerFilter(container, "folder") : - new SimpleFilter(); - filter.addCondition(_pathColumn, srcPath); - return new TableSelector(_table, filter, null).exists(); - } -} +/* + * Copyright (c) 2013-2019 LabKey Corporation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.labkey.api.files; + +import org.apache.commons.lang3.StringUtils; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.jetbrains.annotations.NotNull; +import org.jetbrains.annotations.Nullable; +import org.labkey.api.data.ColumnInfo; +import org.labkey.api.data.CompareType; +import org.labkey.api.data.Container; +import org.labkey.api.data.DbSchema; +import org.labkey.api.data.SQLFragment; +import org.labkey.api.data.SimpleFilter; +import org.labkey.api.data.Sort; +import org.labkey.api.data.SqlExecutor; +import org.labkey.api.data.Table; +import org.labkey.api.data.TableInfo; +import org.labkey.api.data.TableSelector; +import org.labkey.api.data.dialect.SqlDialect; +import org.labkey.api.security.User; +import org.labkey.api.util.FileUtil; + +import java.io.File; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.Collection; +import java.util.Collections; +import java.util.Date; +import java.util.Set; + +/** + * FileListener implementation that can update tables that store file paths in various flavors (URI, standard OS + * paths, etc). + * User: jeckels + * Date: 11/7/12 + */ +public class TableUpdaterFileListener implements FileListener +{ + private static final Logger LOG = LogManager.getLogger(TableUpdaterFileListener.class); + + public static final String TABLE_ALIAS = "x"; + + private final TableInfo _table; + private final SQLFragment _containerFrag; + private final ColumnInfo _pathColumn; + private final PathGetter _pathGetter; + private final ColumnInfo _keyColumn; + + public interface PathGetter + { + /** @return the string that is expected to be in the database */ + String get(File f); + String get(Path f); + /** @return the file path separator (typically '/' or '\') */ + String getSeparatorSuffix(Path path); + } + + public enum Type implements PathGetter + { + /** Turns the file path into a "file:"-prefixed URI */ + uri + { + @Override + public String get(Path path) + { + return FileUtil.pathToString(path); + } + + @Override + public String get(File f) + { + return FileUtil.uriToString(f.toURI()); + } + + @Override + public String getSeparatorSuffix(Path path) + { + return "/"; + } + }, + + /** Just uses getPath() to turn the File into a String */ + filePath + { + @Override + public String get(Path path) + { + if (FileUtil.hasCloudScheme(path)) + return FileUtil.pathToString(path); + else + return get(path.toFile()); + } + + @Override + public String get(File f) + { + return f.getPath(); + } + + @Override + public String getSeparatorSuffix(Path path) + { + return FileUtil.hasCloudScheme(path) ? "/" : File.separator; + } + }, + + /** field has root of file path */ + fileRootPath + { + @Override + public String get(Path path) + { + return filePath.get(path); + } + + @Override + public String get(File f) + { + return filePath.get(f); + } + + @Override + public String getSeparatorSuffix(Path path) + { + return FileUtil.hasCloudScheme(path) ? "/" : File.separator; + } + }, + + /** Just uses getPath() to turn the File into a String, but replaces all backslashes with forward slashes */ + filePathForwardSlash + { + @Override + public String get(Path path) + { + if (FileUtil.hasCloudScheme(path)) + return FileUtil.pathToString(path); + else + return get(path.toFile()); + } + + @Override + public String get(File f) + { + return f.getPath().replace('\\', '/'); + } + + @Override + public String getSeparatorSuffix(Path path) + { + return "/"; + } + } + } + + public TableUpdaterFileListener(TableInfo table, String pathColumn, PathGetter pathGetter) + { + this(table, pathColumn, pathGetter, null, null); + } + + public TableUpdaterFileListener(TableInfo table, String pathColumn, PathGetter pathGetter, @Nullable String keyColumn) + { + this(table, pathColumn, pathGetter, keyColumn, null); + } + + public TableUpdaterFileListener(TableInfo table, String pathColumnName, PathGetter pathGetter, @Nullable String keyColumnName, @Nullable SQLFragment containerFrag) + { + _table = table; + _pathColumn = table.getColumn(pathColumnName); + if (null == _pathColumn) + throw new IllegalStateException("Column not found: " + pathColumnName); + _keyColumn = null == keyColumnName ? null : table.getColumn(keyColumnName); + if (null != keyColumnName && null == _keyColumn) + throw new IllegalStateException("Column not found: " + keyColumnName); + _pathGetter = pathGetter; + _containerFrag = containerFrag; + } + + @Override + public String getSourceName() + { + return _table.getSchema().getName() + "." + _table.getName() + "." + _pathColumn.getName(); + } + + public String getSourceSelect(SqlDialect sqlDialect) + { + String schema = sqlDialect.getStringHandler().quoteStringLiteral(_table.getSchema().getName()); + String table = sqlDialect.getStringHandler().quoteStringLiteral(_table.getName()); + String column = sqlDialect.getStringHandler().quoteStringLiteral(_pathColumn.getName()); + return schema + "." + table + "." + column; + } + + @Override + public void fileCreated(@NotNull File created, @Nullable User user, @Nullable Container container) + { + } + + @Override + public void fileCreated(@NotNull Path created, @Nullable User user, @Nullable Container container) + { + } + + @Override + public int fileMoved(@NotNull File src, @NotNull File dest, @Nullable User user, @Nullable Container container) + { + return fileMoved(src.toPath(), dest.toPath(), user, container); + } + + @Override + public int fileMoved(@NotNull Path src, @NotNull Path dest, @Nullable User user, @Nullable Container container) + { + DbSchema schema = _table.getSchema(); + SqlDialect dialect = schema.getSqlDialect(); + + // Build up SQL that can be used for both the file and any children + SQLFragment sharedSQL = new SQLFragment("UPDATE "); + sharedSQL.append(_table); + sharedSQL.append(_table.getSqlDialect().isSqlServer() ? " WITH (UPDLOCK)" : ""); + sharedSQL.append(" SET "); + if (_table.getColumn("Modified") != null) + { + sharedSQL.append("Modified = ?, "); + sharedSQL.add(new Date()); + } + if (_table.getColumn("ModifiedBy") != null && user != null) + { + sharedSQL.append("ModifiedBy = ?, "); + sharedSQL.add(user.getUserId()); + } + sharedSQL.appendIdentifier(_pathColumn.getSelectIdentifier()); + sharedSQL.append(" = "); + + String srcPath = getSourcePath(src, container); + String destPath = _pathGetter.get(dest); + String srcPathWithout = null; + + // If it's a directory, prep versions with and without a trailing slash. Check the dest because it's already moved by the time this fires + if (srcPath.endsWith("/") || srcPath.endsWith("\\") || Files.isDirectory(dest)) + { + srcPath = getWithSeparator(src, container); + srcPathWithout = getWithoutSeparator(srcPath); + destPath = getWithoutSeparator(destPath); + } + + // Now build up the SQL to handle this specific path + SQLFragment singleEntrySQL = new SQLFragment(sharedSQL); + singleEntrySQL.append("? WHERE ("); + singleEntrySQL.appendIdentifier(_pathColumn.getSelectIdentifier()); + singleEntrySQL.append(" = ?"); + singleEntrySQL.add(destPath); + singleEntrySQL.add(srcPath); + if (null != srcPathWithout) + { + singleEntrySQL.append(" OR "); + singleEntrySQL.appendIdentifier(_pathColumn.getSelectIdentifier()); + singleEntrySQL.append(" = ?"); + singleEntrySQL.add(srcPathWithout); + } + singleEntrySQL.append(")"); + + int rows = schema.getScope().executeWithRetry(tx -> new SqlExecutor(schema).execute(singleEntrySQL)); + LOG.info("Updated " + rows + " row in " + _table + " for move from " + src + " to " + dest); + + // Handle updating child paths, unless we know that the entry is a file. If it's not (either it's a + // directory or it doesn't exist), then try to fix up child records + if ((!Files.exists(dest) || Files.isDirectory(dest))) + { + String separatorSuffix = _pathGetter.getSeparatorSuffix(dest); + if (!srcPath.endsWith(separatorSuffix)) + { + srcPath = srcPath + separatorSuffix; + } + if (!destPath.endsWith(separatorSuffix)) + { + destPath = destPath + separatorSuffix; + } + + int childRowsUpdated = 0; + + // Consider paths with file:///... + if (srcPath.startsWith("file:")) + { + srcPath = "file://" + srcPath.replaceFirst("^file:/+", "/"); + } + SQLFragment whereClause = new SQLFragment(" WHERE "); + whereClause.append(dialect.getStringIndexOfFunction(new SQLFragment("?", srcPath), _pathColumn.getSelectIdentifier().getSql())).append(" = 1"); + + // Make the SQL to handle children + SQLFragment childPathsSQL = new SQLFragment(sharedSQL); + childPathsSQL.append(dialect.concatenate(new SQLFragment("?", destPath), new SQLFragment(dialect.getSubstringFunction(_pathColumn.getSelectIdentifier().getSql(), new SQLFragment(Integer.toString(srcPath.length() + 1)), new SQLFragment("5000"))))); + childPathsSQL.append(whereClause); + childRowsUpdated += new SqlExecutor(schema).execute(childPathsSQL); + + LOG.info("Updated " + childRowsUpdated + " child paths in " + _table + " rows for move from " + src + " to " + dest); + return childRowsUpdated; + } + return 0; + } + + @NotNull + private String getWithSeparator(Path path, Container container) + { + String result = getSourcePath(path, container); + String separator = _pathGetter.getSeparatorSuffix(path); + return result + (result.endsWith(separator) ? "" : separator); + } + + @NotNull + private String getWithoutSeparator(String path) + { + return (path.endsWith("/") || path.endsWith("\\")) ? path.substring(0, path.length() - 1): path; + } + + @Override + public Collection listFiles(@Nullable Container container) + { + Set columns = Collections.singleton(_pathColumn.getName()); + SimpleFilter filter = new SimpleFilter(); + filter.addCondition(_pathColumn, null, CompareType.NONBLANK); + if (container != null) + { + ColumnInfo containerColumn = _table.getColumn("container"); + if (containerColumn == null) + containerColumn = _table.getColumn("folder"); + + if (containerColumn != null) + filter.addCondition(containerColumn, container.getEntityId()); + else + filter.addCondition(new SimpleFilter.SQLClause("1 = 0", null)); + } + + Sort sort = new Sort(_pathColumn.getFieldKey()); + TableSelector selector = new TableSelector(_table, columns, filter, sort); + + selector.setMaxRows(Table.ALL_ROWS); + return selector.getArrayList(File.class); + } + + @Override + public SQLFragment listFilesQuery() + { + return listFilesQuery(false, null, false); + } + + public SQLFragment listFilesQuery(boolean skipCreatedModified, String filePath, boolean extractName) + { + SQLFragment selectFrag = new SQLFragment(); + selectFrag.append("SELECT\n"); + + if (_containerFrag != null) + selectFrag.append("(").append(_containerFrag).append(") AS Container,\n"); + else if (_table.getColumn("Container") != null) + selectFrag.append(" Container,\n"); + else if (_table.getColumn("Folder") != null) + selectFrag.append(" Folder AS Container,\n"); + else + selectFrag.append(" NULL AS Container,\n"); + + if (!skipCreatedModified) + { + if (_table.getColumn("Created") != null) + selectFrag.append(" Created,\n"); + else + selectFrag.append(" NULL AS Created,\n"); + + if (_table.getColumn("CreatedBy") != null) + selectFrag.append(" CreatedBy,\n"); + else + selectFrag.append(" NULL AS CreatedBy,\n"); + + if (_table.getColumn("Modified") != null) + selectFrag.append(" Modified,\n"); + else + selectFrag.append(" NULL AS Modified,\n"); + + if (_table.getColumn("ModifiedBy") != null) + selectFrag.append(" ModifiedBy,\n"); + else + selectFrag.append(" NULL AS ModifiedBy,\n"); + } + + selectFrag.append(" ").appendIdentifier(_pathColumn.getSelectIdentifier()).append(" AS FilePath,\n"); + + if (extractName) + { + SqlDialect dialect = _table.getSchema().getSqlDialect(); + SQLFragment fileNameFrag = new SQLFragment(); + fileNameFrag.append("regexp_replace(").appendIdentifier(_pathColumn.getSelectIdentifier()).append(", "); + fileNameFrag.append(dialect.getStringHandler().quoteStringLiteral(".*/")).append(", "); + fileNameFrag.append(dialect.getStringHandler().quoteStringLiteral("")).append(")");; + selectFrag.append(" ").append(fileNameFrag).append(" AS FilePathShort,\n"); + } + + if (_keyColumn != null) + selectFrag.append(" ").appendIdentifier(_keyColumn.getSelectIdentifier()).append(" AS SourceKey,\n"); + else + selectFrag.append(" NULL AS SourceKey,\n"); + + //selectFrag.append(" ? AS SourceName\n").add(getName()); + selectFrag.append(" ").appendValue(getSourceSelect(_table.getSchema().getSqlDialect())).append(" AS SourceName\n"); + + selectFrag.append("FROM ").append(_table, TABLE_ALIAS).append("\n"); + selectFrag.append("WHERE ").appendIdentifier(_pathColumn.getSelectIdentifier()); + + if (StringUtils.isEmpty(filePath)) + selectFrag.append(" IS NOT NULL\n"); + else + selectFrag.append(" = ").appendStringLiteral(filePath, _table.getSchema().getSqlDialect()).append("\n"); + + return selectFrag; + } + + @NotNull + private String getSourcePath(Path path, Container container) + { + // For uri pathGetter, check that file path exists in table, looking for legacy as well + if (Type.uri == _pathGetter) + { + String srcPath = _pathGetter.get(path); + if (pathExists(srcPath, container)) + return srcPath; + + if (!FileUtil.hasCloudScheme(path)) + { + srcPath = path.toFile().getPath(); // File path format (/users/...) + if (pathExists(srcPath, container)) + return srcPath; + } + + // use original if none found, for directories + } + + return _pathGetter.get(path); + } + + private boolean pathExists(String srcPath, Container container) + { + + SimpleFilter filter = (null != _table.getColumn("container")) ? + SimpleFilter.createContainerFilter(container) : + (null != _table.getColumn("folder")) ? + SimpleFilter.createContainerFilter(container, "folder") : + new SimpleFilter(); + filter.addCondition(_pathColumn, srcPath); + return new TableSelector(_table, filter, null).exists(); + } +} diff --git a/experiment/src/org/labkey/experiment/FileLinkFileListener.java b/experiment/src/org/labkey/experiment/FileLinkFileListener.java index 4a19b2b4ccb..8c0d1c5b729 100644 --- a/experiment/src/org/labkey/experiment/FileLinkFileListener.java +++ b/experiment/src/org/labkey/experiment/FileLinkFileListener.java @@ -1,319 +1,338 @@ -/* - * Copyright (c) 2013-2019 LabKey Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.labkey.experiment; - -import org.apache.commons.lang3.StringUtils; -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; -import org.jetbrains.annotations.NotNull; -import org.jetbrains.annotations.Nullable; -import org.labkey.api.data.ColumnInfo; -import org.labkey.api.data.Container; -import org.labkey.api.data.DbSchema; -import org.labkey.api.data.DbSchemaType; -import org.labkey.api.data.SQLFragment; -import org.labkey.api.data.SqlExecutor; -import org.labkey.api.data.SqlSelector; -import org.labkey.api.data.TableInfo; -import org.labkey.api.data.dialect.SqlDialect; -import org.labkey.api.exp.OntologyManager; -import org.labkey.api.exp.PropertyType; -import org.labkey.api.exp.api.StorageProvisioner; -import org.labkey.api.exp.property.Domain; -import org.labkey.api.exp.property.PropertyService; -import org.labkey.api.files.FileListener; -import org.labkey.api.files.TableUpdaterFileListener; -import org.labkey.api.security.User; -import org.labkey.api.util.FileUtil; -import org.labkey.experiment.api.ExpMaterialTableImpl; -import org.labkey.experiment.api.SampleTypeServiceImpl; - -import java.io.File; -import java.nio.file.Files; -import java.nio.file.Path; -import java.util.ArrayList; -import java.util.Collection; -import java.util.concurrent.atomic.AtomicInteger; - -import static org.labkey.api.exp.api.SampleTypeDomainKind.PROVISIONED_SCHEMA_NAME; - -/** - * Handles fixup of paths stored in OntologyManager for File Link fields. These values are persisted as absolute paths - * on the server's file system. - * User: jeckels - * Date: 11/8/12 - */ -public class FileLinkFileListener implements FileListener -{ - private static final Logger LOG = LogManager.getLogger(FileLinkFileListener.class); - - @Override - public String getSourceName() - { - return "FileLinkFileListener"; - } - - @Override - public void fileCreated(@NotNull File created, @Nullable User user, @Nullable Container container) - { - } - - @Override - public void fileCreated(@NotNull Path created, @Nullable User user, @Nullable Container container) - { - } - - @Override - public int fileMoved(@NotNull File srcFile, @NotNull File destFile, @Nullable User user, @Nullable Container container) - { - return fileMoved(srcFile.toPath(), destFile.toPath(), user, container); - } - - @Override - public int fileMoved(@NotNull Path srcFile, @NotNull Path destFile, @Nullable User user, @Nullable Container container) - { - int result = updateObjectProperty(srcFile, destFile); - - result += updateHardTables(srcFile, destFile, user, container); - return result; - } - - /** Migrate FileLink values stored in exp.ObjectProperty */ - private int updateObjectProperty(Path srcFile, Path destFile) - { - String srcPath = FileUtil.hasCloudScheme(srcFile) ? FileUtil.pathToString(srcFile) : srcFile.toFile().getPath(); - String destPath = FileUtil.hasCloudScheme(destFile) ? FileUtil.pathToString(destFile) : destFile.toFile().getPath(); - - SqlDialect dialect = OntologyManager.getSqlDialect(); - - // Build up SQL that can be used for both the file and any children - SQLFragment sharedSQL = new SQLFragment("UPDATE "); - sharedSQL.append(OntologyManager.getTinfoObjectProperty()); - sharedSQL.append(" SET StringValue = "); - - SQLFragment standardWhereSQL = new SQLFragment(" WHERE PropertyId IN (SELECT PropertyId FROM "); - standardWhereSQL.append(OntologyManager.getTinfoPropertyDescriptor(), "pd"); - standardWhereSQL.append(" WHERE RangeURI = ?)"); - standardWhereSQL.add(PropertyType.FILE_LINK.getTypeUri()); - - // Now build up the SQL to handle this specific path - SQLFragment singleEntrySQL = new SQLFragment(sharedSQL); - singleEntrySQL.append("? "); - singleEntrySQL.add(destPath); - singleEntrySQL.append(standardWhereSQL); - singleEntrySQL.append(" AND StringValue = ?"); - singleEntrySQL.add(srcPath); - - int rows = new SqlExecutor(OntologyManager.getExpSchema()).execute(singleEntrySQL); - LOG.info("Updated " + rows + " row in exp.ObjectProperty for move from " + srcFile + " to " + destFile); - if (rows > 0) - { - // Clear potential object values - OntologyManager.clearPropertyCache(); - } - - // Skip attempting to fix up child paths if we know that the entry is a file. If it's not (either it's a - // directory or it doesn't exist), then try to fix up child records - if (Files.isDirectory(destFile)) - { - if (!srcPath.endsWith(File.separator)) - { - srcPath = srcPath + File.separator; - } - if (!destPath.endsWith(File.separator)) - { - destPath = destPath + File.separator; - } - - // Make the SQL to handle children - SQLFragment childPathsSQL = new SQLFragment(sharedSQL); - childPathsSQL.append(dialect.concatenate(new SQLFragment("?", destPath), new SQLFragment(dialect.getSubstringFunction("StringValue", Integer.toString(srcPath.length() + 1), "5000")))); - childPathsSQL.append(standardWhereSQL); - childPathsSQL.append(" AND "); - childPathsSQL.append(dialect.getStringIndexOfFunction(new SQLFragment("?", srcPath), new SQLFragment("StringValue"))); - childPathsSQL.append(" = 1"); - - int childRows = new SqlExecutor(OntologyManager.getExpSchema()).execute(childPathsSQL); - if (childRows > 0) - { - // Clear potential object values - OntologyManager.clearPropertyCache(); - } - LOG.info("Updated " + childRows + " child paths in exp.ObjectProperty rows for move from " + srcFile + " to " + destFile); - return rows + childRows; - } - return rows; - } - - private int updateHardTables(final Path srcFile, final Path destFile, final User user, final Container container) - { - AtomicInteger result = new AtomicInteger(0); - hardTableFileLinkColumns((schema, table, pathColumn, containerId, domainUri) -> { - // Migrate any paths that match this file move - TableUpdaterFileListener updater = new TableUpdaterFileListener(table, pathColumn.getColumnName(), TableUpdaterFileListener.Type.filePath); - int movedCount = updater.fileMoved(srcFile, destFile, user, container); - result.addAndGet(movedCount); - - if (movedCount > 0 && schema.getName().equalsIgnoreCase(PROVISIONED_SCHEMA_NAME)) - ExpMaterialTableImpl.refreshMaterializedView(domainUri, SampleTypeServiceImpl.SampleChangeType.update); - }); - return result.intValue(); - } - - private interface ForEachFileLinkColumn - { - void exec(DbSchema schema, TableInfo table, ColumnInfo pathColumn, String containerId, @Nullable String domainUri); - } - - private void hardTableFileLinkColumns(final ForEachFileLinkColumn block) - { - // Figure out all of the FileLink columns in hard tables managed by OntologyManager - SQLFragment sql = new SQLFragment("SELECT dd.Container, dd.DomainId, dd.StorageTableName, dd.StorageSchemaName, pd.Name, pd.StorageColumnName FROM "); - sql.append(OntologyManager.getTinfoDomainDescriptor(), "dd"); - sql.append(", "); - sql.append(OntologyManager.getTinfoPropertyDescriptor(), "pd"); - sql.append(", "); - sql.append(OntologyManager.getTinfoPropertyDomain(), "m"); - sql.append(" WHERE dd.DomainId = m.DomainId AND pd.PropertyId = m.PropertyId AND pd.RangeURI = ? "); - sql.add(PropertyType.FILE_LINK.getTypeUri()); - sql.append(" AND dd.StorageTableName IS NOT NULL AND dd.StorageSchemaName IS NOT NULL AND pd.Name IS NOT NULL"); - - new SqlSelector(OntologyManager.getExpSchema(), sql).forEachMap(row -> { - // Find the DbSchema/TableInfo/ColumnInfo for the FileLink column - String storageSchemaName = row.get("StorageSchemaName").toString(); - DbSchema schema = DbSchema.get(storageSchemaName, DbSchemaType.Provisioned); - Domain domain = PropertyService.get().getDomain((Integer) row.get("DomainId")); - // Issue 50781: LKSM: File fields in Sample Types sometimes showing as Text Fields - // Don't use schema.getTable(storageTableName); - if (domain != null) - { - TableInfo tableInfo = StorageProvisioner.get().getSchemaTableInfo(domain); - if (tableInfo != null) - { - String containerId = row.get("Container").toString(); - if (containerId != null) - { - ColumnInfo pathCol = tableInfo.getColumn(row.get("Name").toString()); - // Issue 53502: also try to get tableInfo column by StorageColumnName if not found by Name - if (pathCol == null) - pathCol = tableInfo.getColumn(row.get("StorageColumnName").toString()); - - if (pathCol != null) - block.exec(schema, tableInfo, pathCol, containerId, domain.getTypeURI()); - } - } - } - }); - } - - @Override - public Collection listFiles(@Nullable Container container) - { - Collection files = new ArrayList<>(); - - files.addAll(listObjectPropertyFiles(container)); - files.addAll(listHardTableFiles(container)); - - return files; - } - - private Collection listObjectPropertyFiles(@Nullable Container container) - { - SQLFragment frag = new SQLFragment(); - frag.append("SELECT StringValue\n"); - frag.append("FROM\n"); - frag.append(" ").append(OntologyManager.getTinfoObjectProperty(), "op").append(",\n"); - frag.append(" ").append(OntologyManager.getTinfoObject(), "o").append("\n"); - frag.append("WHERE "); - frag.append(" o.ObjectId = op.ObjectId AND\n"); - if (container != null) - { - frag.append(" o.Container = ? AND \n"); - frag.add(container); - } - frag.append(" PropertyId IN (\n"); - frag.append(" SELECT PropertyId\n"); - frag.append(" FROM ").append(OntologyManager.getTinfoPropertyDescriptor(), "pd").append("\n"); - frag.append(" WHERE RangeURI = ?\n").add(PropertyType.FILE_LINK.getTypeUri()); - frag.append(" )\n"); - - SqlSelector selector = new SqlSelector(OntologyManager.getExpSchema(), frag); - return selector.getArrayList(File.class); - } - - private Collection listHardTableFiles(@NotNull final Container container) - { - final Collection files = new ArrayList<>(); - hardTableFileLinkColumns((schema, table, pathColumn, containerId, domainUri) -> { - TableUpdaterFileListener updater = new TableUpdaterFileListener(table, pathColumn.getColumnName(), TableUpdaterFileListener.Type.filePath); - files.addAll(updater.listFiles(container)); - }); - return files; - } - - @Override - public SQLFragment listFilesQuery() - { - return listFilesQuery(false); - } - - public SQLFragment listFilesQuery(boolean skipCreatedModified) - { - return listFilesQuery(skipCreatedModified, null); - } - - public SQLFragment listFilesQuery(boolean skipCreatedModified, String filePath) - { - final SQLFragment frag = new SQLFragment(); - - // Object property files - frag.append("SELECT\n"); - frag.append(" o.Container,\n"); - if (!skipCreatedModified) - { - frag.append(" NULL AS Created,\n"); - frag.append(" NULL AS CreatedBy,\n"); - frag.append(" NULL AS Modified,\n"); - frag.append(" NULL AS ModifiedBy,\n"); - } - frag.append(" op.StringValue AS FilePath,\n"); - frag.append(" o.ObjectId AS SourceKey,\n"); - frag.append(" 'exp.object' AS SourceName\n"); - frag.append("FROM\n"); - frag.append(" ").append(OntologyManager.getTinfoObjectProperty(), "op").append(",\n"); - frag.append(" ").append(OntologyManager.getTinfoObject(), "o").append("\n"); - frag.append("WHERE\n"); - if (StringUtils.isEmpty(filePath)) - frag.append(" op.StringValue IS NOT NULL AND\n"); - else - frag.append(" op.StringValue = ").appendStringLiteral(filePath, OntologyManager.getTinfoObject().getSqlDialect()).append(" AND\n"); - frag.append(" o.ObjectId = op.ObjectId AND\n"); - frag.append(" PropertyId IN (\n"); - frag.append(" SELECT PropertyId\n"); - frag.append(" FROM ").append(OntologyManager.getTinfoPropertyDescriptor(), "pd").append("\n"); - frag.append(" WHERE RangeURI = ?\n").add(PropertyType.FILE_LINK.getTypeUri()); - frag.append(" )\n"); - - hardTableFileLinkColumns((schema, table, pathColumn, containerId, domainUri) -> { - SQLFragment containerFrag = new SQLFragment("?", containerId); - TableUpdaterFileListener updater = new TableUpdaterFileListener(table, pathColumn.getColumnName(), TableUpdaterFileListener.Type.filePath, null, containerFrag); - frag.append("UNION").append(StringUtils.isEmpty(filePath) ? "" : " ALL" /*keep duplicate*/).append("\n"); - frag.append(updater.listFilesQuery(skipCreatedModified, filePath)); - }); - - return frag; - } -} +/* + * Copyright (c) 2013-2019 LabKey Corporation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.labkey.experiment; + +import org.apache.commons.lang3.StringUtils; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.jetbrains.annotations.NotNull; +import org.jetbrains.annotations.Nullable; +import org.labkey.api.data.ColumnInfo; +import org.labkey.api.data.Container; +import org.labkey.api.data.DbSchema; +import org.labkey.api.data.DbSchemaType; +import org.labkey.api.data.SQLFragment; +import org.labkey.api.data.SqlExecutor; +import org.labkey.api.data.SqlSelector; +import org.labkey.api.data.TableInfo; +import org.labkey.api.data.dialect.SqlDialect; +import org.labkey.api.exp.OntologyManager; +import org.labkey.api.exp.PropertyType; +import org.labkey.api.exp.api.StorageProvisioner; +import org.labkey.api.exp.property.Domain; +import org.labkey.api.exp.property.PropertyService; +import org.labkey.api.files.FileListener; +import org.labkey.api.files.TableUpdaterFileListener; +import org.labkey.api.security.User; +import org.labkey.api.util.FileUtil; +import org.labkey.experiment.api.ExpMaterialTableImpl; +import org.labkey.experiment.api.SampleTypeServiceImpl; + +import java.io.File; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.Collection; +import java.util.concurrent.atomic.AtomicInteger; + +import static org.labkey.api.exp.api.SampleTypeDomainKind.PROVISIONED_SCHEMA_NAME; + +/** + * Handles fixup of paths stored in OntologyManager for File Link fields. These values are persisted as absolute paths + * on the server's file system. + * User: jeckels + * Date: 11/8/12 + */ +public class FileLinkFileListener implements FileListener +{ + private static final Logger LOG = LogManager.getLogger(FileLinkFileListener.class); + + @Override + public String getSourceName() + { + return "FileLinkFileListener"; + } + + @Override + public void fileCreated(@NotNull File created, @Nullable User user, @Nullable Container container) + { + } + + @Override + public void fileCreated(@NotNull Path created, @Nullable User user, @Nullable Container container) + { + } + + @Override + public int fileMoved(@NotNull File srcFile, @NotNull File destFile, @Nullable User user, @Nullable Container container) + { + return fileMoved(srcFile.toPath(), destFile.toPath(), user, container); + } + + @Override + public int fileMoved(@NotNull Path srcFile, @NotNull Path destFile, @Nullable User user, @Nullable Container container) + { + int result = updateObjectProperty(srcFile, destFile); + + result += updateHardTables(srcFile, destFile, user, container); + return result; + } + + /** Migrate FileLink values stored in exp.ObjectProperty */ + private int updateObjectProperty(Path srcFile, Path destFile) + { + String srcPath = FileUtil.hasCloudScheme(srcFile) ? FileUtil.pathToString(srcFile) : srcFile.toFile().getPath(); + String destPath = FileUtil.hasCloudScheme(destFile) ? FileUtil.pathToString(destFile) : destFile.toFile().getPath(); + + SqlDialect dialect = OntologyManager.getSqlDialect(); + + // Build up SQL that can be used for both the file and any children + SQLFragment sharedSQL = new SQLFragment("UPDATE "); + sharedSQL.append(OntologyManager.getTinfoObjectProperty()); + sharedSQL.append(" SET StringValue = "); + + SQLFragment standardWhereSQL = new SQLFragment(" WHERE PropertyId IN (SELECT PropertyId FROM "); + standardWhereSQL.append(OntologyManager.getTinfoPropertyDescriptor(), "pd"); + standardWhereSQL.append(" WHERE RangeURI = ?)"); + standardWhereSQL.add(PropertyType.FILE_LINK.getTypeUri()); + + // Now build up the SQL to handle this specific path + SQLFragment singleEntrySQL = new SQLFragment(sharedSQL); + singleEntrySQL.append("? "); + singleEntrySQL.add(destPath); + singleEntrySQL.append(standardWhereSQL); + singleEntrySQL.append(" AND StringValue = ?"); + singleEntrySQL.add(srcPath); + + int rows = new SqlExecutor(OntologyManager.getExpSchema()).execute(singleEntrySQL); + LOG.info("Updated " + rows + " row in exp.ObjectProperty for move from " + srcFile + " to " + destFile); + if (rows > 0) + { + // Clear potential object values + OntologyManager.clearPropertyCache(); + } + + // Skip attempting to fix up child paths if we know that the entry is a file. If it's not (either it's a + // directory or it doesn't exist), then try to fix up child records + if (Files.isDirectory(destFile)) + { + if (!srcPath.endsWith(File.separator)) + { + srcPath = srcPath + File.separator; + } + if (!destPath.endsWith(File.separator)) + { + destPath = destPath + File.separator; + } + + // Make the SQL to handle children + SQLFragment childPathsSQL = new SQLFragment(sharedSQL); + childPathsSQL.append(dialect.concatenate(new SQLFragment("?", destPath), new SQLFragment(dialect.getSubstringFunction("StringValue", Integer.toString(srcPath.length() + 1), "5000")))); + childPathsSQL.append(standardWhereSQL); + childPathsSQL.append(" AND "); + childPathsSQL.append(dialect.getStringIndexOfFunction(new SQLFragment("?", srcPath), new SQLFragment("StringValue"))); + childPathsSQL.append(" = 1"); + + int childRows = new SqlExecutor(OntologyManager.getExpSchema()).execute(childPathsSQL); + if (childRows > 0) + { + // Clear potential object values + OntologyManager.clearPropertyCache(); + } + LOG.info("Updated " + childRows + " child paths in exp.ObjectProperty rows for move from " + srcFile + " to " + destFile); + return rows + childRows; + } + return rows; + } + + private int updateHardTables(final Path srcFile, final Path destFile, final User user, final Container container) + { + AtomicInteger result = new AtomicInteger(0); + hardTableFileLinkColumns((schema, table, pathColumn, containerId, domainUri) -> { + // Migrate any paths that match this file move + TableUpdaterFileListener updater = new TableUpdaterFileListener(table, pathColumn.getColumnName(), TableUpdaterFileListener.Type.filePath); + int movedCount = updater.fileMoved(srcFile, destFile, user, container); + result.addAndGet(movedCount); + + if (movedCount > 0 && schema.getName().equalsIgnoreCase(PROVISIONED_SCHEMA_NAME)) + ExpMaterialTableImpl.refreshMaterializedView(domainUri, SampleTypeServiceImpl.SampleChangeType.update); + }); + return result.intValue(); + } + + private interface ForEachFileLinkColumn + { + void exec(DbSchema schema, TableInfo table, ColumnInfo pathColumn, String containerId, @Nullable String domainUri); + } + + private void hardTableFileLinkColumns(final ForEachFileLinkColumn block) + { + // Figure out all of the FileLink columns in hard tables managed by OntologyManager + SQLFragment sql = new SQLFragment("SELECT dd.Container, dd.DomainId, dd.StorageTableName, dd.StorageSchemaName, pd.Name, pd.StorageColumnName FROM "); + sql.append(OntologyManager.getTinfoDomainDescriptor(), "dd"); + sql.append(", "); + sql.append(OntologyManager.getTinfoPropertyDescriptor(), "pd"); + sql.append(", "); + sql.append(OntologyManager.getTinfoPropertyDomain(), "m"); + sql.append(" WHERE dd.DomainId = m.DomainId AND pd.PropertyId = m.PropertyId AND pd.RangeURI = ? "); + sql.add(PropertyType.FILE_LINK.getTypeUri()); + sql.append(" AND dd.StorageTableName IS NOT NULL AND dd.StorageSchemaName IS NOT NULL AND pd.Name IS NOT NULL"); + + new SqlSelector(OntologyManager.getExpSchema(), sql).forEachMap(row -> { + // Find the DbSchema/TableInfo/ColumnInfo for the FileLink column + String storageSchemaName = row.get("StorageSchemaName").toString(); + DbSchema schema = DbSchema.get(storageSchemaName, DbSchemaType.Provisioned); + Domain domain = PropertyService.get().getDomain((Integer) row.get("DomainId")); + // Issue 50781: LKSM: File fields in Sample Types sometimes showing as Text Fields + // Don't use schema.getTable(storageTableName); + if (domain != null) + { + TableInfo tableInfo = StorageProvisioner.get().getSchemaTableInfo(domain); + if (tableInfo != null) + { + String containerId = row.get("Container").toString(); + if (containerId != null) + { + ColumnInfo pathCol = tableInfo.getColumn(row.get("Name").toString()); + // Issue 53502: also try to get tableInfo column by StorageColumnName if not found by Name + if (pathCol == null) + pathCol = tableInfo.getColumn(row.get("StorageColumnName").toString()); + + if (pathCol != null) + block.exec(schema, tableInfo, pathCol, containerId, domain.getTypeURI()); + } + } + } + }); + } + + @Override + public Collection listFiles(@Nullable Container container) + { + Collection files = new ArrayList<>(); + + files.addAll(listObjectPropertyFiles(container)); + files.addAll(listHardTableFiles(container)); + + return files; + } + + private Collection listObjectPropertyFiles(@Nullable Container container) + { + SQLFragment frag = new SQLFragment(); + frag.append("SELECT StringValue\n"); + frag.append("FROM\n"); + frag.append(" ").append(OntologyManager.getTinfoObjectProperty(), "op").append(",\n"); + frag.append(" ").append(OntologyManager.getTinfoObject(), "o").append("\n"); + frag.append("WHERE "); + frag.append(" o.ObjectId = op.ObjectId AND\n"); + if (container != null) + { + frag.append(" o.Container = ? AND \n"); + frag.add(container); + } + frag.append(" PropertyId IN (\n"); + frag.append(" SELECT PropertyId\n"); + frag.append(" FROM ").append(OntologyManager.getTinfoPropertyDescriptor(), "pd").append("\n"); + frag.append(" WHERE RangeURI = ?\n").add(PropertyType.FILE_LINK.getTypeUri()); + frag.append(" )\n"); + + SqlSelector selector = new SqlSelector(OntologyManager.getExpSchema(), frag); + return selector.getArrayList(File.class); + } + + private Collection listHardTableFiles(@NotNull final Container container) + { + final Collection files = new ArrayList<>(); + hardTableFileLinkColumns((schema, table, pathColumn, containerId, domainUri) -> { + TableUpdaterFileListener updater = new TableUpdaterFileListener(table, pathColumn.getColumnName(), TableUpdaterFileListener.Type.filePath); + files.addAll(updater.listFiles(container)); + }); + return files; + } + + @Override + public SQLFragment listFilesQuery() + { + return listFilesQuery(false); + } + + public SQLFragment listFilesQuery(boolean skipCreatedModified) + { + return listFilesQuery(skipCreatedModified, null); + } + + public SQLFragment listFilesQuery(boolean skipCreatedModified, String filePath) + { + final SQLFragment frag = new SQLFragment(); + + // Object property files + frag.append("SELECT\n"); + frag.append(" o.Container,\n"); + if (!skipCreatedModified) + { + frag.append(" NULL AS Created,\n"); + frag.append(" NULL AS CreatedBy,\n"); + frag.append(" NULL AS Modified,\n"); + frag.append(" NULL AS ModifiedBy,\n"); + } + frag.append(" op.StringValue AS FilePath,\n"); + frag.append(" o.ObjectId AS SourceKey,\n"); + frag.append(" 'exp.object' AS SourceName\n"); + frag.append("FROM\n"); + frag.append(" ").append(OntologyManager.getTinfoObjectProperty(), "op").append(",\n"); + frag.append(" ").append(OntologyManager.getTinfoObject(), "o").append("\n"); + frag.append("WHERE\n"); + if (StringUtils.isEmpty(filePath)) + frag.append(" op.StringValue IS NOT NULL AND\n"); + else + frag.append(" op.StringValue = ").appendStringLiteral(filePath, OntologyManager.getTinfoObject().getSqlDialect()).append(" AND\n"); + frag.append(" o.ObjectId = op.ObjectId AND\n"); + frag.append(" PropertyId IN (\n"); + frag.append(" SELECT PropertyId\n"); + frag.append(" FROM ").append(OntologyManager.getTinfoPropertyDescriptor(), "pd").append("\n"); + frag.append(" WHERE RangeURI = ?\n").add(PropertyType.FILE_LINK.getTypeUri()); + frag.append(" )\n"); + + hardTableFileLinkColumns((schema, table, pathColumn, containerId, domainUri) -> { + SQLFragment containerFrag = new SQLFragment("?", containerId); + TableUpdaterFileListener updater = new TableUpdaterFileListener(table, pathColumn.getColumnName(), TableUpdaterFileListener.Type.filePath, null, containerFrag); + frag.append("UNION").append(StringUtils.isEmpty(filePath) ? "" : " ALL" /*keep duplicate*/).append("\n"); + frag.append(updater.listFilesQuery(skipCreatedModified, filePath, false)); + }); + + return frag; + } + + @Override + public SQLFragment listSampleFilesQuery() + { + final SQLFragment frag = new SQLFragment(); + + hardTableFileLinkColumns((schema, table, pathColumn, containerId, domainUri) -> { + if (schema.getName().equals("expsampleset")) + { + SQLFragment containerFrag = new SQLFragment("?", containerId); + TableUpdaterFileListener updater = new TableUpdaterFileListener(table, pathColumn.getColumnName(), TableUpdaterFileListener.Type.filePath, "rowid", containerFrag); + if (!frag.isEmpty()) + frag.append("UNION").append("").append("\n"); + frag.append(updater.listFilesQuery(true, null, true)); + } + }); + + return frag; + } +} diff --git a/filecontent/src/org/labkey/filecontent/FileContentServiceImpl.java b/filecontent/src/org/labkey/filecontent/FileContentServiceImpl.java index f3ed5d2eaff..6d732c778a5 100644 --- a/filecontent/src/org/labkey/filecontent/FileContentServiceImpl.java +++ b/filecontent/src/org/labkey/filecontent/FileContentServiceImpl.java @@ -1,1953 +1,1974 @@ -/* - * Copyright (c) 2009-2019 LabKey Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.labkey.filecontent; - -import org.apache.commons.io.FilenameUtils; -import org.apache.commons.lang3.StringUtils; -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; -import org.jetbrains.annotations.NotNull; -import org.jetbrains.annotations.Nullable; -import org.junit.After; -import org.junit.Assert; -import org.junit.Test; -import org.labkey.api.action.SpringActionController; -import org.labkey.api.admin.AdminUrls; -import org.labkey.api.attachments.AttachmentDirectory; -import org.labkey.api.cache.Cache; -import org.labkey.api.cache.CacheManager; -import org.labkey.api.cloud.CloudStoreService; -import org.labkey.api.collections.CaseInsensitiveHashMap; -import org.labkey.api.data.CompareType; -import org.labkey.api.data.Container; -import org.labkey.api.data.ContainerManager; -import org.labkey.api.data.ContainerManager.ContainerListener; -import org.labkey.api.data.ContainerType; -import org.labkey.api.data.CoreSchema; -import org.labkey.api.data.SQLFragment; -import org.labkey.api.data.SimpleFilter; -import org.labkey.api.data.TabContainerType; -import org.labkey.api.data.Table; -import org.labkey.api.data.TableInfo; -import org.labkey.api.data.TableSelector; -import org.labkey.api.data.WorkbookContainerType; -import org.labkey.api.exp.Lsid; -import org.labkey.api.exp.api.ExpData; -import org.labkey.api.exp.api.ExpProtocol; -import org.labkey.api.exp.api.ExpRun; -import org.labkey.api.exp.api.ExperimentService; -import org.labkey.api.exp.query.ExpDataTable; -import org.labkey.api.files.DirectoryPattern; -import org.labkey.api.files.FileContentService; -import org.labkey.api.files.FileListener; -import org.labkey.api.files.FileRoot; -import org.labkey.api.files.FilesAdminOptions; -import org.labkey.api.files.MissingRootDirectoryException; -import org.labkey.api.files.UnsetRootDirectoryException; -import org.labkey.api.files.view.FilesWebPart; -import org.labkey.api.module.Module; -import org.labkey.api.module.ModuleLoader; -import org.labkey.api.pipeline.PipeRoot; -import org.labkey.api.pipeline.PipelineService; -import org.labkey.api.pipeline.PipelineUrls; -import org.labkey.api.query.BatchValidationException; -import org.labkey.api.query.FieldKey; -import org.labkey.api.query.QueryUpdateService; -import org.labkey.api.security.User; -import org.labkey.api.security.permissions.AdminOperationsPermission; -import org.labkey.api.settings.AppProps; -import org.labkey.api.settings.RandomSiteSettingsPropertyHandler; -import org.labkey.api.settings.StartupPropertyEntry; -import org.labkey.api.settings.WriteableAppProps; -import org.labkey.api.test.TestWhen; -import org.labkey.api.util.ContainerUtil; -import org.labkey.api.util.DOM; -import org.labkey.api.util.FileUtil; -import org.labkey.api.util.GUID; -import org.labkey.api.util.HtmlString; -import org.labkey.api.util.NetworkDrive; -import org.labkey.api.util.PageFlowUtil; -import org.labkey.api.util.Path; -import org.labkey.api.util.TestContext; -import org.labkey.api.util.URIUtil; -import org.labkey.api.view.ActionURL; -import org.labkey.api.view.HttpView; -import org.labkey.api.view.ViewBackgroundInfo; -import org.labkey.api.view.ViewContext; -import org.labkey.api.view.template.WarningProvider; -import org.labkey.api.view.template.WarningService; -import org.labkey.api.view.template.Warnings; -import org.labkey.api.webdav.WebdavResource; -import org.labkey.api.webdav.WebdavService; - -import java.beans.PropertyChangeEvent; -import java.io.BufferedWriter; -import java.io.File; -import java.io.IOException; -import java.net.URI; -import java.net.URISyntaxException; -import java.nio.file.Files; -import java.nio.file.InvalidPathException; -import java.nio.file.StandardOpenOption; -import java.util.ArrayList; -import java.util.Collection; -import java.util.Collections; -import java.util.Date; -import java.util.HashMap; -import java.util.HashSet; -import java.util.LinkedHashMap; -import java.util.LinkedHashSet; -import java.util.List; -import java.util.Map; -import java.util.Set; -import java.util.concurrent.CopyOnWriteArrayList; -import java.util.regex.Pattern; -import java.util.stream.Stream; - -import static org.labkey.api.settings.AppProps.SCOPE_SITE_SETTINGS; -import static org.labkey.api.util.DOM.Attribute.href; -import static org.labkey.api.util.DOM.at; - -public class FileContentServiceImpl implements FileContentService, WarningProvider -{ - private static final Logger _log = LogManager.getLogger(FileContentServiceImpl.class); - private static final String UPLOAD_LOG = ".upload.log"; - private static final FileContentServiceImpl INSTANCE = new FileContentServiceImpl(); - - private final ContainerListener _containerListener = new FileContentServiceContainerListener(); - private final List _fileListeners = new CopyOnWriteArrayList<>(); - - private final List _ziploaderPattern = new CopyOnWriteArrayList<>(); - - private volatile boolean _fileRootSetViaStartupProperty = false; - private String _problematicFileRootMessage; - - enum FileAction - { - UPLOAD, - DELETE - } - - static FileContentServiceImpl getInstance() - { - return INSTANCE; - } - - private FileContentServiceImpl() - { - WarningService.get().register(this); - } - - @Override - @NotNull - public List getContainersForFilePath(java.nio.file.Path path) - { - // Ignore cloud files for now - if (FileUtil.hasCloudScheme(path)) - return Collections.emptyList(); - - // If the path is under the default root, do optimistic simple match for containers under the default root - File defaultRoot = getSiteDefaultRoot(); - java.nio.file.Path defaultRootPath = defaultRoot.toPath(); - if (path.startsWith(defaultRootPath)) - { - java.nio.file.Path rel = defaultRootPath.relativize(path); - if (rel.getNameCount() > 0) - { - Container root = ContainerManager.getRoot(); - Container next = root; - while (rel.getNameCount() > 0) - { - // check if there exists a child container that matches the next path segment - java.nio.file.Path top = rel.subpath(0, 1); - assert top != null; - Container child = next.getChild(top.getFileName().toString()); - if (child == null) - break; - - next = child; - - if(rel.getNameCount() > 1) - { - rel = rel.subpath(1, rel.getNameCount()); - } - else - { - break; - } - } - - if (next != null && !next.equals(root)) - { - // verify our naive file path is correct for the container -- it may have a file root other than the default - java.nio.file.Path fileRoot = getFileRootPath(next); - if (fileRoot != null && path.startsWith(fileRoot)) - return Collections.singletonList(next); - } - } - } - - // TODO: Create cache of file root and pipeline root paths -> list of containers - - return Collections.emptyList(); - } - - @Override - public @Nullable File getFileRoot(@NotNull Container c, @NotNull ContentType type) - { - switch (type) - { - case files: - case assayfiles: - String folderName = getFolderName(type); - if (folderName == null) - folderName = ""; - - java.nio.file.Path dir = getFileRootPath(c); - return dir != null ? dir.resolve(folderName).toFile() : null; - - case pipeline: - PipeRoot root = PipelineService.get().findPipelineRoot(c); - return root != null ? root.getRootPath() : null; - } - return null; - } - - @Override - public @Nullable java.nio.file.Path getFileRootPath(@NotNull Container c, @NotNull ContentType type) - { - switch (type) - { - case files: - case assayfiles: - java.nio.file.Path fileRootPath = getFileRootPath(c); - if (null != fileRootPath && !FileUtil.hasCloudScheme(fileRootPath)) // Don't add @files when we're in the cloud - fileRootPath = fileRootPath.resolve(getFolderName(type)); - return fileRootPath; - - case pipeline: - PipeRoot root = PipelineService.get().findPipelineRoot(c); - return root != null ? root.getRootNioPath() : null; - } - return null; - } - - // Returns full uri to file root for this container. filePath is optional relative path to a file under the file root - @Override - public @Nullable URI getFileRootUri(@NotNull Container c, @NotNull ContentType type, @Nullable String filePath) - { - java.nio.file.Path root = FileContentService.get().getFileRootPath(c, FileContentService.ContentType.files); - if (root != null) - { - String path = root.toString(); - if (filePath != null) { - path += filePath; - } - - // non-unix needs a leading slash - if (!path.startsWith("/") && !path.startsWith("\\")) - { - path = "/" + path; - } - return FileUtil.createUri(path); - } - - return null; - } - - @Override - public @Nullable File getFileRoot(@NotNull Container c) - { - java.nio.file.Path path = getFileRootPath(c); - throwIfPathNotFile(path, c); - return path.toFile(); - } - - @Override - public @Nullable java.nio.file.Path getFileRootPath(@NotNull Container c) - { - if (c == null) - return null; - - if (c.isRoot()) - { - return getSiteDefaultRootPath(); - } - - if (!isFileRootDisabled(c)) - { - FileRoot root = FileRootManager.get().getFileRoot(c); - - // check if there is a site wide file root - if (root.getPath() == null || isUseDefaultRoot(c)) - { - return getDefaultRootPath(c, true); - } - else - return getNioPath(c, root.getPath()); - } - return null; - } - - @Override - public File getDefaultRoot(Container c, boolean createDir) - { - return getDefaultRootPath(c, createDir).toFile(); - } - - @Override - public java.nio.file.Path getDefaultRootPath(@NotNull Container c, boolean createDir) - { - Container firstOverride = getFirstAncestorWithOverride(c); - - java.nio.file.Path parentRoot; - if (firstOverride == null) - { - parentRoot = getSiteDefaultRoot().toPath(); - firstOverride = ContainerManager.getRoot(); - } - else - { - parentRoot = getFileRootPath(firstOverride); - } - - if (parentRoot != null && firstOverride != null) - { - java.nio.file.Path fileRootPath; - if (FileUtil.hasCloudScheme(parentRoot)) - { - // For cloud root, we don't have to create directories for this path - fileRootPath = CloudStoreService.get().getPathForOtherContainer(firstOverride, c, FileUtil.pathToString(parentRoot), new Path("")); - } - else - { - // For local, the path may be several directories deep (since it matches the LK folder path), so we should create the directories for that path - fileRootPath = FileUtil.appendPath(parentRoot.toFile(), Path.parse(getRelativePath(c, firstOverride))).toPath(); - - try - { - if (createDir && !NetworkDrive.exists(fileRootPath)) - FileUtil.createDirectories(fileRootPath); - } - catch (IOException e) - { - return null; // throw new RuntimeException(e); TODO: does returning null make certain tests, like TargetedMSQCGuideSetTest pass on Windows? - } - } - - return fileRootPath; - } - return null; - } - - // Return pretty path string for defaultFileRoot and boolean true if defaultFileRoot is cloud - @Override - public DefaultRootInfo getDefaultRootInfo(Container container) - { - String defaultRoot = ""; - boolean isDefaultRootCloud = false; - java.nio.file.Path defaultRootPath = getDefaultRootPath(container, false); - String cloudName = null; - if (defaultRootPath != null) - { - isDefaultRootCloud = FileUtil.hasCloudScheme(defaultRootPath); - if (isDefaultRootCloud && !container.isProject()) - { - FileRoot fileRoot = getDefaultFileRoot(container); - if (null != fileRoot) - defaultRoot = fileRoot.getPath(); - if (null != defaultRoot) - cloudName = getCloudRootName(defaultRoot); - } - else - { - defaultRoot = FileUtil.getAbsolutePath(container, defaultRootPath.toUri()); - } - } - return new DefaultRootInfo(defaultRootPath, defaultRoot, isDefaultRootCloud, cloudName); - } - - @Nullable - // Get FileRoot associated with path returned form getDefaultRootPath() - public FileRoot getDefaultFileRoot(Container c) - { - Container firstOverride = getFirstAncestorWithOverride(c); - - if (firstOverride == null) - firstOverride = ContainerManager.getRoot(); - - if (null != firstOverride) - return FileRootManager.get().getFileRoot(firstOverride); - return null; - } - - private @NotNull String getRelativePath(Container c, Container ancestor) - { - return c.getPath().replaceAll("^" + Pattern.quote(ancestor.getPath()), ""); - } - - //returns the first parent container that has a custom file root, or NULL if none have overrides - private Container getFirstAncestorWithOverride(Container c) - { - Container toTest = c.getParent(); - if (toTest == null) - return null; - - while (isUseDefaultRoot(toTest)) - { - if (toTest == null || toTest.equals(ContainerManager.getRoot())) - return null; - - toTest = toTest.getParent(); - } - - return toTest; - } - - private java.nio.file.Path getNioPath(Container c, @NotNull String fileRootPath) - { - if (isCloudFileRoot(fileRootPath)) - return CloudStoreService.get().getPath(c, getCloudRootName(fileRootPath), new org.labkey.api.util.Path("")); - - return FileUtil.stringToPath(c, fileRootPath, false); // fileRootPath is unencoded - } - - private boolean isCloudFileRoot(String fileRootPseudoPath) - { - return StringUtils.startsWith(fileRootPseudoPath, FileContentService.CLOUD_ROOT_PREFIX); - } - - private String getCloudRootName(@NotNull String fileRootPseudoPath) - { - return fileRootPseudoPath.substring(fileRootPseudoPath.indexOf(FileContentService.CLOUD_ROOT_PREFIX) + FileContentService.CLOUD_ROOT_PREFIX.length() + 1); - } - - @Override - public boolean isCloudRoot(Container c) - { - if (null != c) - { - java.nio.file.Path fileRootPath = getFileRootPath(c); - return null != fileRootPath && FileUtil.hasCloudScheme(fileRootPath); - } - return false; - } - - @Override - @NotNull - public String getCloudRootName(Container c) - { - if (null != c) - { - if (isCloudRoot(c)) - { - FileRoot root = FileRootManager.get().getFileRoot(c); - if (null == root.getPath() || isUseDefaultRoot(c)) - { - Container firstOverride = getFirstAncestorWithOverride(c); - if (null == firstOverride) - firstOverride = ContainerManager.getRoot(); - root = FileRootManager.get().getFileRoot(firstOverride); - if (null == root.getPath()) - return ""; - } - return getCloudRootName(root.getPath()); - } - } - return ""; - } - - @Override - public void setCloudRoot(@NotNull Container c, String cloudRootName) - { - _setFileRoot(c, FileContentService.CLOUD_ROOT_PREFIX + "/" + cloudRootName); - } - - @Override - public void setFileRoot(@NotNull Container c, @Nullable File path) - { - _setFileRoot(c, (null != path ? FileUtil.getAbsoluteCaseSensitiveFile(path).getAbsolutePath() : null)); - } - - @Override - public void setFileRootPath(@NotNull Container c, @Nullable String strPath) - { - String absolutePath = null; - if (strPath != null) - { - URI uri = FileUtil.createUri(strPath, false); // strPath is unencoded - if (FileUtil.hasCloudScheme(uri)) - absolutePath = FileUtil.getAbsolutePath(c, uri); - else - absolutePath = FileUtil.getAbsoluteCaseSensitiveFile(new File(uri)).getAbsolutePath(); - } - _setFileRoot(c, absolutePath); - } - - private void _setFileRoot(@NotNull Container c, @Nullable String absolutePath) - { - if (!c.isContainerFor(ContainerType.DataType.fileRoot)) - throw new IllegalArgumentException("File roots cannot be set for containers of type " + c.getContainerType().getName()); - - FileRoot root = FileRootManager.get().getFileRoot(c); - root.setEnabled(true); - - String oldValue = root.getPath(); - String newValue = null; - - // clear out the root - if (absolutePath == null) - root.setPath(null); - else - { - root.setPath(absolutePath); - newValue = root.getPath(); - } - - FileRootManager.get().saveFileRoot(null, root); - ContainerManager.ContainerPropertyChangeEvent evt = new ContainerManager.ContainerPropertyChangeEvent( - c, ContainerManager.Property.WebRoot, oldValue, newValue); - ContainerManager.firePropertyChangeEvent(evt); - } - - @Override - public void disableFileRoot(Container container) - { - if (container == null || container.isRoot()) - throw new IllegalArgumentException("Disabling either a null project or the root project is not allowed."); - - Container effective = container.getContainerFor(ContainerType.DataType.fileRoot); - if (effective != null) - { - FileRoot root = FileRootManager.get().getFileRoot(effective); - String oldValue = root.getPath(); - root.setEnabled(false); - FileRootManager.get().saveFileRoot(null, root); - - ContainerManager.ContainerPropertyChangeEvent evt = new ContainerManager.ContainerPropertyChangeEvent( - container, ContainerManager.Property.WebRoot, oldValue, null); - ContainerManager.firePropertyChangeEvent(evt); - } - } - - @Override - public boolean isFileRootDisabled(Container c) - { - Container effective = c.getContainerFor(ContainerType.DataType.fileRoot); - if (null == effective) - return false; - - FileRoot root = FileRootManager.get().getFileRoot(effective); - return !root.isEnabled(); - } - - @Override - public boolean isUseDefaultRoot(Container c) - { - if (c == null) - return true; - - Container effective = c.getContainerFor(ContainerType.DataType.fileRoot); - if (null == effective) - return true; - - FileRoot root = FileRootManager.get().getFileRoot(effective); - return root.isUseDefault() || StringUtils.isEmpty(root.getPath()); - } - - @Override - public void setIsUseDefaultRoot(Container c, boolean useDefaultRoot) - { - Container effective = c.getContainerFor(ContainerType.DataType.fileRoot); - if (effective != null) - { - FileRoot root = FileRootManager.get().getFileRoot(effective); - String oldValue = root.getPath(); - root.setEnabled(true); - root.setUseDefault(useDefaultRoot); - if (useDefaultRoot) - root.setPath(null); - FileRootManager.get().saveFileRoot(null, root); - - ContainerManager.ContainerPropertyChangeEvent evt = new ContainerManager.ContainerPropertyChangeEvent( - effective, ContainerManager.Property.WebRoot, oldValue, null); - ContainerManager.firePropertyChangeEvent(evt); - } - } - - @Override - public @NotNull java.nio.file.Path getSiteDefaultRootPath() - { - return getSiteDefaultRoot().toPath(); - } - - @Override - public @NotNull File getSiteDefaultRoot() - { - // Site default is always on file system - File root = AppProps.getInstance().getFileSystemRoot(); - - try - { - if (!NetworkDrive.exists(root)) - { - File configuredRoot = root; - root = getDefaultRoot(); - if (configuredRoot != null && !configuredRoot.equals(root)) - { - String message = "The configured site-wide file root " + configuredRoot + " does not exist. Falling back to " + root; - if (!message.equals(_problematicFileRootMessage)) - { - _problematicFileRootMessage = message; - _log.error(_problematicFileRootMessage); - } - } - } - else - { - _problematicFileRootMessage = null; - } - - if (!NetworkDrive.exists(root)) - { - if (FileUtil.mkdirs(root)) - { - _log.info("Created site-wide file root " + root); - } - else - { - _log.error("Failed when attempting to create site-wide file root " + root); - } - } - } - catch (IOException e) - { - throw new RuntimeException("Unable to create file root directory", e); - } - - return root; - } - - @Override - public String getProblematicFileRootMessage() - { - return _problematicFileRootMessage; - } - - private @NotNull File getDefaultRoot() throws IOException - { - File explodedPath = ModuleLoader.getInstance().getCoreModule().getExplodedPath(); - - File root = explodedPath.getParentFile(); - if (root != null) - { - if (root.getParentFile() != null) - root = root.getParentFile(); - } - File defaultRoot = new File(root, "files"); - if (!NetworkDrive.exists(defaultRoot)) - FileUtil.mkdirs(defaultRoot); - - return defaultRoot; - } - - @Override - public void setSiteDefaultRoot(File root, User user) - { - if (root == null) - throw new IllegalArgumentException("Invalid site root: specified root is null"); - - if (!NetworkDrive.exists(root)) - throw new IllegalArgumentException("Invalid site root: " + root.getAbsolutePath() + " does not exist"); - - File prevRoot = getSiteDefaultRoot(); - WriteableAppProps props = AppProps.getWriteableInstance(); - - props.setFileSystemRoot(root.getAbsolutePath()); - props.save(user); - - FileRootManager.get().clearCache(); - ContainerManager.ContainerPropertyChangeEvent evt = new ContainerManager.ContainerPropertyChangeEvent( - ContainerManager.getRoot(), ContainerManager.Property.SiteRoot, prevRoot, root); - ContainerManager.firePropertyChangeEvent(evt); - } - - @Override - public void setWebfilesEnabled(boolean enabled, User user) - { - WriteableAppProps props = AppProps.getWriteableInstance(); - props.setWebfilesEnabled(enabled); - props.save(user); - } - - @Override - public FileSystemAttachmentParent registerDirectory(Container c, String name, String path, boolean relative) - { - FileSystemAttachmentParent parent = new FileSystemAttachmentParent(); - parent.setContainer(c); - if (null == name) - name = path; - parent.setName(name); - parent.setPath(path); - parent.setRelative(relative); - //We do this because insert does not return new fields - parent.setEntityid(GUID.makeGUID()); - - FileSystemAttachmentParent ret = Table.insert(HttpView.currentContext().getUser(), CoreSchema.getInstance().getMappedDirectories(), parent); - ContainerManager.ContainerPropertyChangeEvent evt = new ContainerManager.ContainerPropertyChangeEvent( - c, ContainerManager.Property.AttachmentDirectory, null, ret); - ContainerManager.firePropertyChangeEvent(evt); - return ret; - } - - @Override - public void unregisterDirectory(Container c, String name) - { - FileSystemAttachmentParent parent = getRegisteredDirectory(c, name); - SimpleFilter filter = SimpleFilter.createContainerFilter(c); - filter.addCondition(FieldKey.fromParts("Name"), name); - Table.delete(CoreSchema.getInstance().getMappedDirectories(), filter); - ContainerManager.ContainerPropertyChangeEvent evt = new ContainerManager.ContainerPropertyChangeEvent( - c, ContainerManager.Property.AttachmentDirectory, parent, null); - ContainerManager.firePropertyChangeEvent(evt); - } - - @Override - public @Nullable AttachmentDirectory getMappedAttachmentDirectory(Container c, boolean createDir) throws UnsetRootDirectoryException, MissingRootDirectoryException - { - return getMappedAttachmentDirectory(c, ContentType.files, createDir); - } - - @Override - @Nullable - public AttachmentDirectory getMappedAttachmentDirectory(Container c, ContentType contentType, boolean createDir) throws UnsetRootDirectoryException - { - try - { - if (createDir) //force create - getMappedDirectory(c, true); - else if (null == getMappedDirectory(c, false)) - return null; - - return new FileSystemAttachmentParent(c, contentType); - } - catch (IOException e) - { - _log.error("Cannot get mapped directory for " + c.getPath(), e); - return null; - } - } - - public java.nio.file.Path getMappedDirectory(Container c, boolean create) throws UnsetRootDirectoryException, IOException - { - java.nio.file.Path root = getFileRootPath(c); - if (!FileUtil.hasCloudScheme(root)) - { - if (null == root) - { - if (create) - throw new UnsetRootDirectoryException(c.isRoot() ? c : c.getProject()); - else - return null; - } - - if (!NetworkDrive.exists(root)) - { - if (create) - throw new MissingRootDirectoryException(c.isRoot() ? c : c.getProject(), root); - else - return null; - - } - } - return root; - } - - @Override - public FileSystemAttachmentParent getRegisteredDirectory(Container c, String name) - { - SimpleFilter filter = SimpleFilter.createContainerFilter(c); - filter.addCondition(FieldKey.fromParts("Name"), name); - - return new TableSelector(CoreSchema.getInstance().getMappedDirectories(), filter, null).getObject(FileSystemAttachmentParent.class); - } - - @Override - public FileSystemAttachmentParent getRegisteredDirectoryFromEntityId(Container c, String entityId) - { - SimpleFilter filter = SimpleFilter.createContainerFilter(c); - filter.addCondition(FieldKey.fromParts("EntityId"), entityId); - - return new TableSelector(CoreSchema.getInstance().getMappedDirectories(), filter, null).getObject(FileSystemAttachmentParent.class); - } - - @Override - public @NotNull Collection getRegisteredDirectories(Container c) - { - SimpleFilter filter = SimpleFilter.createContainerFilter(c); - - return Collections.unmodifiableCollection(new TableSelector(CoreSchema.getInstance().getMappedDirectories(), filter, null).getCollection(FileSystemAttachmentParent.class)); - } - - private class FileContentServiceContainerListener implements ContainerListener - { - @Override - public void containerCreated(Container c, User user) - { - try - { - // Will create directory if it's a default dir - getMappedDirectory(c, false); - } - catch (IOException ex) - { - /* */ - } - } - - @Override - public void containerDeleted(Container c, User user) - { - java.nio.file.Path dir = null; - try - { - // don't delete the file contents if they have a project override - if (isUseDefaultRoot(c) && !isCloudRoot(c)) // Don't do anything for cloud root here. CloudContainerListener will handle - dir = getMappedDirectory(c, false); - - if (null != dir) - { - FileUtil.deleteDir(dir); - } - } - catch (Exception e) - { - _log.error("containerDeleted", e); - } - - ContainerUtil.purgeTable(CoreSchema.getInstance().getMappedDirectories(), c, null); - } - - @Override - public void containerMoved(Container c, Container oldParent, User user) - { - /* **** Cases: - SRC DEST - specific local path same -- no work - specific cloud path same -- no work - local default local default -- move tree - local default cloud default -- move tree - cloud default local default -- move tree - cloud default cloud default -- if change bucket, move tree - *************************************************************/ - if (isUseDefaultRoot(c)) - { - java.nio.file.Path srcParent = getFileRootPath(oldParent); - java.nio.file.Path dest = getFileRootPath(c); - if (null != srcParent && null != dest) - { - if (!FileUtil.hasCloudScheme(srcParent)) - { - File src = new File(srcParent.toFile(), c.getName()); - if (NetworkDrive.exists(src)) - { - if (!FileUtil.hasCloudScheme(dest)) - { - // local -> local - moveFileRoot(src, dest.toFile(), user, c); - } - else - { - // local -> cloud; source starts under @files - File filesSrc = FileUtil.appendName(src, FILES_LINK); - if (NetworkDrive.exists(filesSrc)) - moveFileRoot(filesSrc.toPath(), dest, user, c); - FileUtil.deleteDir(src); // moveFileRoot will delete @files, but we need to delete its parent - } - } - } - else - { - // Get source path using moving container and parent's config (cloudRoot), because that config must be the source config - java.nio.file.Path src = CloudStoreService.get().getPath(c, getCloudRootName(oldParent), new Path("")); - if (!FileUtil.hasCloudScheme(dest)) - { - // cloud -> local; destination is under @files - dest = dest.resolve(FILES_LINK); - moveFileRoot(src, dest, user, c); - } - else - { - // cloud -> cloud - if (!getCloudRootName(oldParent).equals(getCloudRootName(c))) - { - // Different configs - moveFileRoot(src, dest, user, c); - } - } - } - } - } - } - - @Override - public void propertyChange(PropertyChangeEvent propertyChangeEvent) - { - ContainerManager.ContainerPropertyChangeEvent evt = (ContainerManager.ContainerPropertyChangeEvent)propertyChangeEvent; - Container c = evt.container; - - switch (evt.property) - { - case Name: // container rename event - { - String oldValue = (String) propertyChangeEvent.getOldValue(); - String newValue = (String) propertyChangeEvent.getNewValue(); - - java.nio.file.Path location; - try - { - location = getMappedDirectory(c, false); - if (location != null && !FileUtil.hasCloudScheme(location)) // If cloud, folder name for container not dependent on Name - { - //Don't rely on container object. Seems not to point to the - //new location even AFTER rename. Just construct new file paths - File locationFile = location.toFile(); - File parentDir = locationFile.getParentFile(); - File oldLocation = new File(parentDir, oldValue); - File newLocation = new File(parentDir, newValue); - if (NetworkDrive.exists(newLocation)) - moveToDeleted(newLocation); - - if (NetworkDrive.exists(oldLocation)) - { - oldLocation.renameTo(newLocation); - fireFileMoveEvent(oldLocation, newLocation, evt.user, evt.container); - } - } - } - catch (IOException ex) - { - _log.error(ex); - } - - break; - } - } - } - } - - - @Override - public @Nullable String getFolderName(FileContentService.ContentType type) - { - if (type != null) - return "@" + type.name(); - return null; - } - - - /** - * Move the file or directory into a ".deleted" directory under the parent directory. - * @return True if successfully moved. - */ - private static boolean moveToDeleted(File fileToMove) throws IOException - { - if (!NetworkDrive.exists(fileToMove)) - return false; - - File parent = fileToMove.getParentFile(); - - File deletedDir = new File(parent, ".deleted"); - if (!NetworkDrive.exists(deletedDir)) - if (!FileUtil.mkdir(deletedDir)) - return false; - - File newLocation = new File(deletedDir, fileToMove.getName()); - if (NetworkDrive.exists(newLocation)) - FileUtil.deleteDir(newLocation); - - return fileToMove.renameTo(newLocation); - } - - static void logFileAction(java.nio.file.Path directory, String fileName, FileAction action, User user) - { - try (BufferedWriter fw = Files.newBufferedWriter(directory.resolve(UPLOAD_LOG), StandardOpenOption.APPEND, StandardOpenOption.CREATE)) - { - fw.write(action.toString() + "\t" + fileName + "\t" + new Date() + "\t" + (user == null ? "(unknown)" : user.getEmail()) + "\n"); - } - catch (Exception x) - { - //Just log it. - _log.error(x); - } - } - - @Override - public FilesAdminOptions getAdminOptions(Container c) - { - FileRoot root = FileRootManager.get().getFileRoot(c); - String xml = null; - - if (!StringUtils.isBlank(root.getProperties())) - { - xml = root.getProperties(); - } - return new FilesAdminOptions(c, xml); - } - - @Override - public void setAdminOptions(Container c, FilesAdminOptions options) - { - if (options != null) - { - setAdminOptions(c, options.serialize()); - } - } - - @Override - public void setAdminOptions(Container c, String properties) - { - FileRoot root = FileRootManager.get().getFileRoot(c); - - root.setProperties(properties); - FileRootManager.get().saveFileRoot(null, root); - } - - public static final String NAMESPACE_PREFIX = "FileProperties"; - public static final String PROPERTIES_DOMAIN = "File Properties"; - public static final String TYPE_PROPERTIES = "FileProperties"; - - @Override - public String getDomainURI(Container container) - { - return getDomainURI(container, getAdminOptions(container).getFileConfig()); - } - - @Override - public String getDomainURI(Container container, FilesAdminOptions.fileConfig config) - { - while (config == FilesAdminOptions.fileConfig.useParent && container != container.getParent()) - { - container = container.getParent(); - config = getAdminOptions(container).getFileConfig(); - } - - //String typeURI = "urn:lsid:" + AppProps.getInstance().getDefaultLsidAuthority() + ":List" + ".Folder-" + container.getRowId() + ":" + name; - - return new Lsid("urn:lsid:labkey.com:" + NAMESPACE_PREFIX + ".Folder-" + container.getRowId() + ':' + TYPE_PROPERTIES).toString(); - } - - @Override @Nullable - public ExpData getDataObject(WebdavResource resource, Container c) - { - return getDataObject(resource, c, null, false); - } - - @Nullable - private static ExpData getDataObject(WebdavResource resource, Container c, User user, boolean create) - { - // TODO: S3: seems to only be called from Search and currently we're not searching in cloud. SaveCustomPropsAction seems unused - if (resource != null) - { - File file = resource.getFile(); - if (file != null) - { - ExpData data = ExperimentService.get().getExpDataByURL(file, c); - - if (data == null && create) - { - data = ExperimentService.get().createData(c, FileContentService.UPLOADED_FILE); - data.setName(file.getName()); - data.setDataFileURI(file.toURI()); - data.save(user); - } - return data; - } - } - return null; - } - - @Override - public QueryUpdateService getFilePropsUpdateService(TableInfo tinfo, Container container) - { - return new FileQueryUpdateService(tinfo, container); - } - - @Override - public boolean isValidProjectRoot(String root) - { - File f = new File(root); - return NetworkDrive.exists(f) && f.isDirectory(); - } - - @Override - public void moveFileRoot(java.nio.file.Path prev, java.nio.file.Path dest, @Nullable User user, @Nullable Container container) - { - if (!FileUtil.hasCloudScheme(prev) && !FileUtil.hasCloudScheme(dest)) - { - moveFileRoot(prev.toFile(), dest.toFile(), user, container); // Both files; try rename - } - else - { - try - { - // At least one is in the cloud - FileUtil.copyDirectory(prev, dest); - FileUtil.deleteDir(prev); // TODO use more efficient delete - fireFileMoveEvent(prev, dest, user, container); - } - catch (IOException e) - { - _log.error("error occurred moving the file root", e); - } - } - } - - @Override - public void moveFileRoot(File prev, File dest, @Nullable User user, @Nullable Container container) - { - try - { - _log.info("moving " + prev.getPath() + " to " + dest.getPath()); - boolean doRename = true; - - // Our best bet for perf is to do a rename, which doesn't require creating an actual copy. - // If it exists, try deleting the target directory, which will only succeed if it's empty, but would - // enable using renameTo() method. Don't delete if it's a symbolic link, since it wouldn't be recreated - // in the same way. - if (NetworkDrive.exists(dest) && !Files.isSymbolicLink(dest.toPath())) - doRename = dest.delete(); - - if (doRename && !prev.renameTo(dest)) - { - _log.info("rename failed, attempting to copy"); - - //listFiles can return null, which could cause a NPE - File[] children = prev.listFiles(); - if (children != null) - { - for (File file : children) - FileUtil.copyBranch(file, dest); - } - FileUtil.deleteDir(prev); - } - fireFileMoveEvent(prev, dest, user, container); - } - catch (IOException e) - { - _log.error("error occurred moving the file root", e); - } - } - - @Override - public void fireFileCreateEvent(@NotNull File created, @Nullable User user, @Nullable Container container) - { - fireFileCreateEvent(created.toPath(), user, container); - } - - @Override - public void fireFileCreateEvent(@NotNull java.nio.file.Path created, @Nullable User user, @Nullable Container container) - { - java.nio.file.Path absPath = FileUtil.getAbsoluteCaseSensitivePath(container, created); - for (FileListener fileListener : _fileListeners) - { - fileListener.fileCreated(absPath, user, container); - } - } - - @Override - public void fireFileReplacedEvent(@NotNull java.nio.file.Path replaced, @Nullable User user, @Nullable Container container) - { - java.nio.file.Path absPath = FileUtil.getAbsoluteCaseSensitivePath(container, replaced); - for (FileListener fileListener : _fileListeners) - { - fileListener.fileReplaced(absPath, user, container); - } - } - - @Override - public void fireFileDeletedEvent(@NotNull java.nio.file.Path deleted, @Nullable User user, @Nullable Container container) - { - java.nio.file.Path absPath = FileUtil.getAbsoluteCaseSensitivePath(container, deleted); - for (FileListener fileListener : _fileListeners) - { - fileListener.fileDeleted(absPath, user, container); - } - } - - @Override - public int fireFileMoveEvent(@NotNull File src, @NotNull File dest, @Nullable User user, @Nullable Container container) - { - return fireFileMoveEvent(src.toPath(), dest.toPath(), user, container); - } - - @Override - public int fireFileMoveEvent(@NotNull java.nio.file.Path src, @NotNull java.nio.file.Path dest, @Nullable User user, @Nullable Container container) - { - return fireFileMoveEvent(src, dest, user, container, null); - } - - @Override - public int fireFileMoveEvent(@NotNull java.nio.file.Path src, @NotNull java.nio.file.Path dest, @Nullable User user, @Nullable Container sourceContainer, @Nullable Container targetContainer) - { - // Make sure that we've got the best representation of the file that we can - java.nio.file.Path absSrc = FileUtil.getAbsoluteCaseSensitivePath(sourceContainer, src); - java.nio.file.Path absDest = FileUtil.getAbsoluteCaseSensitivePath(targetContainer != null ? targetContainer : sourceContainer, dest); - int result = 0; - for (FileListener fileListener : _fileListeners) - { - result += fileListener.fileMoved(absSrc, absDest, user, sourceContainer, targetContainer); - } - return result; - } - - @Override - public void addFileListener(FileListener listener) - { - _fileListeners.add(listener); - } - - @Override - public Map> listFiles(@NotNull Container container) - { - Map> files = new LinkedHashMap<>(); - for (FileListener fileListener : _fileListeners) - { - files.put(fileListener.getSourceName(), new HashSet<>(fileListener.listFiles(container))); - } - return files; - } - - @Override - public SQLFragment listFilesQuery(@NotNull User currentUser) - { - SQLFragment frag = new SQLFragment(); - if (currentUser == null || !currentUser.hasSiteAdminPermission()) - { - frag.append("SELECT\n"); - frag.append(" CAST(NULL AS VARCHAR) AS Container,\n"); - frag.append(" NULL AS Created,\n"); - frag.append(" NULL AS CreatedBy,\n"); - frag.append(" NULL AS Modified,\n"); - frag.append(" NULL AS ModifiedBy,\n"); - frag.append(" NULL AS FilePath,\n"); - frag.append(" NULL AS SourceKey,\n"); - frag.append(" NULL AS SourceName\n"); - frag.append("WHERE 1 = 0"); - } - else - { - String union = ""; - frag.append("("); - for (FileListener fileListener : _fileListeners) - { - SQLFragment subselect = fileListener.listFilesQuery(); - if (subselect != null) - { - frag.append(union); - frag.append(subselect); - union = "UNION\n"; - } - } - frag.append(")"); - } - return frag; - } - - @Override - public void setFileRootSetViaStartupProperty(boolean fileRootSetViaStartupProperty) - { - _fileRootSetViaStartupProperty = fileRootSetViaStartupProperty; - } - - @Override - public boolean isFileRootSetViaStartupProperty() - { - return _fileRootSetViaStartupProperty; - } - - public ContainerListener getContainerListener() - { - return _containerListener; - } - - public Set> getNodes(boolean isShowOverridesOnly, @Nullable String browseUrl, Container c) - { - Set> children = new LinkedHashSet<>(); - - try { - java.nio.file.Path assayFilesRoot = getFileRootPath(c, ContentType.assayfiles); - if (NetworkDrive.exists(assayFilesRoot)) - { - Map node = createFileSetNode(c, ASSAY_FILES, assayFilesRoot); - node.put("default", false); - node.put("webdavURL", FilesWebPart.getRootPath(c, ASSAY_FILES).toString()); - children.add(node); - } - - AttachmentDirectory root = getMappedAttachmentDirectory(c, false); - if (root != null) - { - boolean isDefault = isUseDefaultRoot(c); - if (!isDefault || !isShowOverridesOnly) - { - ActionURL config = PageFlowUtil.urlProvider(AdminUrls.class).getProjectSettingsFileURL(c); - Map node = createFileSetNode(c, FILES_LINK, root.getFileSystemDirectoryPath()); - node.put("default", isUseDefaultRoot(c)); - node.put("configureURL", config.getEncodedLocalURIString()); - node.put("browseURL", browseUrl); - node.put("webdavURL", FilesWebPart.getRootPath(c, FILES_LINK).toString()); - - children.add(node); - } - } - - for (AttachmentDirectory fileSet : getRegisteredDirectories(c)) - { - ActionURL config = new ActionURL(FileContentController.ShowAdminAction.class, c); - Map node = createFileSetNode(c, fileSet.getName(), fileSet.getFileSystemDirectoryPath()); - node.put("configureURL", config.getEncodedLocalURIString()); - node.put("browseURL", browseUrl); - node.put("webdavURL", FilesWebPart.getRootPath(c, FILE_SETS_LINK, fileSet.getName()).toString()); - node.put("rootType", "fileset"); - - children.add(node); - } - - PipeRoot pipeRoot = PipelineService.get().findPipelineRoot(c); - if (pipeRoot != null) - { - boolean isDefault = PipelineService.get().hasSiteDefaultRoot(c); - if (!isDefault || !isShowOverridesOnly) - { - ActionURL config = PageFlowUtil.urlProvider(PipelineUrls.class).urlSetup(c); - ActionURL pipelineBrowse = PageFlowUtil.urlProvider(PipelineUrls.class).urlBrowse(c, null); - Map node = createFileSetNode(c, PIPELINE_LINK, pipeRoot.getRootNioPath()); - node.put("default", isDefault ); - node.put("configureURL", config.getEncodedLocalURIString()); - node.put("browseURL", pipelineBrowse.getEncodedLocalURIString()); - node.put("webdavURL", FilesWebPart.getRootPath(c, PIPELINE_LINK).toString()); - - children.add(node); - } - } - } - catch (IOException | UnsetRootDirectoryException ignored) {} - return children; - } - - protected Map createFileSetNode(Container container, String name, java.nio.file.Path dir) - { - Map node = new HashMap<>(); - if (dir != null) - { - node.put("name", name); - node.put("path", FileUtil.getAbsolutePath(container, dir)); - node.put("leaf", true); - } - return node; - } - - public String getAbsolutePathFromDataFileUrl(String dataFileUrl, Container container) - { - return FileUtil.getAbsolutePath(container, FileUtil.createUri(dataFileUrl)); - } - - @Nullable - @Override - public URI getWebDavUrl(@NotNull java.nio.file.Path path, @NotNull Container container, @NotNull PathType type) - { - PipeRoot root = PipelineService.get().getPipelineRootSetting(container); - java.nio.file.Path assayFilesPath = getFileRootPath(container, ContentType.assayfiles); - path = path.toAbsolutePath(); - String relPath = null; - URI rootWebDavUrl = null; - - try - { - // currently, only report if the file is under the parent container - if (root != null && root.isUnderRoot(path)) - { - relPath = root.relativePath(path); - rootWebDavUrl = root.getWebdavURL(); - } - else if (assayFilesPath != null && URIUtil.isDescendant(assayFilesPath.toUri(), path.toUri())) - { - relPath = assayFilesPath.relativize(path).toString(); - rootWebDavUrl = FilesWebPart.getRootPath(container, ASSAY_FILES); - } - - if (relPath != null) - { - relPath = Path.parse(FilenameUtils.separatorsToUnix(relPath)).encode(); - - return switch (type) - { - case folderRelative -> new URI(relPath); - case serverRelative -> new URI(rootWebDavUrl + (rootWebDavUrl.getPath().endsWith("/") ? "" : "/") + relPath); - case full -> new URI(AppProps.getInstance().getBaseServerUrl() + rootWebDavUrl + (rootWebDavUrl.getPath().endsWith("/") ? "" : "/") + relPath); - }; - } - } - catch (InvalidPathException | URISyntaxException e) - { - _log.error("Invalid WebDav URL from: " + path, e); - } - - return null; - } - - @Override - public String getDataFileRelativeFileRootPath(@NotNull String dataFileUrl, Container container) - { - Set> children = getNodes(false, null, container); - String filesRoot = null; // the path for @files - for (Map child : children) - { - String rootName = (String) child.get("name"); - String rootPath = (String) child.get("path"); - - // skip default @pipeline, which is the same as @files - if (PIPELINE_LINK.equals(rootName)) - { - if((boolean) child.get("default") || rootPath.equals(filesRoot)) - continue; - } - - if (FILES_LINK.equals(rootName)) - filesRoot = rootPath; - - String absoluteFilePath = getAbsolutePathFromDataFileUrl(dataFileUrl, container); - if (StringUtils.startsWith(absoluteFilePath, rootPath)) - { - String offset = absoluteFilePath.replace(rootPath, "").replace("\\", "/"); - int lastSlash = offset.lastIndexOf("/"); - if (lastSlash <= 0) - return "/"; - else - return offset.substring(0, lastSlash); - } - } - return null; - } - - @Override - public void ensureFileData(@NotNull ExpDataTable table) - { - Container container = table.getUserSchema().getContainer(); - // The current user may not have insert permission, and they didn't necessarily upload the files anyway - User user = User.getAdminServiceUser(); - QueryUpdateService qus = table.getUpdateService(); - if (qus == null) - { - throw new IllegalArgumentException("getUpdateServer() returned null from " + table); - } - - synchronized (_fileDataUpToDateCache) - { - if (_fileDataUpToDateCache.get(container) != null) // already synced in the past 5 minutes, skip - return; - - _fileDataUpToDateCache.put(container, true); - } - - List existingDataFileUrls = getDataFileUrls(container); - Collection filesets = getRegisteredDirectories(container); - Set> children = getNodes(false, null, container); - String filesRoot = null; // the path for @files - for (Map child : children) - { - String rootName = (String) child.get("name"); - String rootPathVal = (String) child.get("path"); - - // skip default @pipeline, which is the same as @files - if (PIPELINE_LINK.equals(rootName)) - { - if((boolean) child.get("default") || rootPathVal.equals(filesRoot)) - continue; - } - - if (FILES_LINK.equals(rootName)) - filesRoot = rootPathVal; - - String rootDavUrl = (String) child.get("webdavURL"); - - WebdavResource resource = getResource(rootDavUrl); - if (resource == null) - continue; - - List> rows = new ArrayList<>(); - BatchValidationException errors = new BatchValidationException(); - File file = resource.getFile(); - - if (file == null) - { - String rootType = (String) child.get("rootType"); - if ("fileset".equals(rootType)) - { - for (AttachmentDirectory fileset : filesets) - { - if (fileset.getName().equals(rootName)) - { - try - { - file = fileset.getFileSystemDirectory(); - } - catch (MissingRootDirectoryException e) - { - _log.error("Unable to list files for fileset: " + rootName, e); - } - break; - } - } - } - } - - if (file == null) - return; - - try (var ignore = SpringActionController.ignoreSqlUpdates()) - { - java.nio.file.Path rootPath = file.toPath(); - - try (Stream pathStream = Files.walk(rootPath, 100)) // prevent symlink loop - { - pathStream - .filter(path -> !Files.isSymbolicLink(path) && path.compareTo(rootPath) != 0) // exclude symlink & root - .forEach(path -> { - if (!containsUrlOrVariation(existingDataFileUrls, path)) - rows.add(new CaseInsensitiveHashMap<>(Collections.singletonMap("DataFileUrl", path.toUri().toString()))); - }); - } - - qus.insertRows(user, container, rows, errors, null, null); - } - catch (Exception e) - { - _log.error("Error listing content of directory: " + file.getAbsolutePath(), e); - } - } - } - - - @Override - public void addZiploaderPattern(DirectoryPattern directoryPattern) - { - _ziploaderPattern.add(directoryPattern); - } - - @Override - public List getZiploaderPatterns(Container container) - { - List registeredPatterns = new ArrayList<>(); - for(Module module : container.getActiveModules()) - { - _ziploaderPattern.forEach(p -> { - if(p.getModule().getName().equalsIgnoreCase(module.getName())) - registeredPatterns.add(p); - }); - } - return registeredPatterns; - } - - public List getDataFileUrls(Container container) - { - SimpleFilter filter = SimpleFilter.createContainerFilter(container); - filter.addCondition(FieldKey.fromParts("DataFileUrl"), null, CompareType.NONBLANK); - TableSelector selector = new TableSelector(ExperimentService.get().getTinfoData(), Collections.singleton("DataFileUrl"), filter, null); - return selector.getArrayList(String.class); - } - - public Path getPath(String uri) - { - Path path = Path.decode(uri); - - if (!path.startsWith(WebdavService.getPath()) && path.contains(WebdavService.getPath().getName())) - { - String newPath = path.toString(); - int idx = newPath.indexOf(WebdavService.getPath().toString()); - - if (idx != -1) - { - newPath = newPath.substring(idx); - path = Path.parse(newPath); - } - } - return path; - } - - @Nullable - public WebdavResource getResource(String uri) - { - Path path = getPath(uri); - return WebdavService.get().getResolver().lookup(path); - } - - public static void throwIfPathNotFile(java.nio.file.Path path, Container container) - { - if (null == path) - { - throw new RuntimeException("No path to evaluate in " + container.getPath()); - } - if (FileUtil.hasCloudScheme(path)) - { - throw new RuntimeException("Cannot get File object from Cloud File Root in " + container.getPath()); - } - } - - private boolean containsUrlOrVariation(List existingUrls, java.nio.file.Path path) - { - String url = path.toUri().toString(); - if (existingUrls.contains(url)) - return true; - - boolean urlHasTrailingSlash = (Files.isDirectory(path) && (url.endsWith("/") || url.endsWith(File.pathSeparator))); - if (urlHasTrailingSlash && existingUrls.contains(url.substring(0, url.length() - 1))) - return true; - - if (!FileUtil.hasCloudScheme(path)) - { - File file = path.toFile(); - String legacyUrl = file.toURI().toString(); - if (existingUrls.contains(legacyUrl)) // Legacy URI format (file:/users/...) - return true; - - return existingUrls.contains(file.getPath()); - } - return false; - } - - @Override - public File getMoveTargetFile(String absoluteFilePath, @NotNull Container sourceContainer, @NotNull Container targetContainer) - { - if (absoluteFilePath == null) - return null; - - File file = new File(absoluteFilePath); - if (!NetworkDrive.exists(file)) - { - _log.warn("File '" + absoluteFilePath + "' not found and cannot be moved"); - return null; - } - - File sourceFileRoot = getFileRoot(sourceContainer); - if (sourceFileRoot == null) - return null; - - String sourceRootPath = sourceFileRoot.getAbsolutePath(); - if (!absoluteFilePath.startsWith(sourceRootPath)) - { - _log.warn("File '" + absoluteFilePath + "' not currently located in source folder '" + sourceRootPath + "'. Not moving."); - return null; - } - File targetFileRoot = getFileRoot(targetContainer); - if (targetFileRoot == null) - return null; - - String targetPath = absoluteFilePath.replace(sourceRootPath, targetFileRoot.getAbsolutePath()); - File targetFile = new File(targetPath); - return FileUtil.findUniqueFileName(file.getName(), targetFile.getParentFile()); - } - - @Override - public void addDynamicWarnings(@NotNull Warnings warnings, @Nullable ViewContext context, boolean showAllWarnings) - { - if (_problematicFileRootMessage != null && context != null && ContainerManager.getRoot().hasPermission(context.getUser(), AdminOperationsPermission.class)) - { - warnings.add(DOM.createHtmlFragment(_problematicFileRootMessage, " ", DOM.A(at(href, PageFlowUtil.urlProvider(AdminUrls.class).getFilesSiteSettingsURL()), "Configure File System Access"))); - } - else if (showAllWarnings) - { - try - { - warnings.add(HtmlString.of("Configured site-wide file root " + getDefaultRoot() + " does not exist. Falling back to " + getDefaultRoot())); - } - catch (IOException ignored) {} - } - } - - // Cache with short-lived entries so that exp.files can perform reasonably - private static final Cache _fileDataUpToDateCache = CacheManager.getCache(CacheManager.UNLIMITED, 5 * CacheManager.MINUTE, "Files"); - - @TestWhen(TestWhen.When.BVT) - public static class TestCase extends AssertionError - { - private static final String TRICKY_CHARACTERS_FOR_PROJECT_NAMES = "\u2603~!@$&()_+{}-=[],.#\u00E4\u00F6\u00FC"; - - private static final String PROJECT1 = "FileRootTestProject1" + TRICKY_CHARACTERS_FOR_PROJECT_NAMES; - private static final String PROJECT1_SUBFOLDER1 = "Subfolder1"; - private static final String PROJECT1_SUBFOLDER2 = "Subfolder2" + TRICKY_CHARACTERS_FOR_PROJECT_NAMES; - private static final String PROJECT1_SUBSUBFOLDER = "SubSubfolder"; - private static final String PROJECT1_SUBSUBFOLDER_SIBLING = "SubSubfolderSibling"; - private static final String PROJECT2 = "FileRootTestProject2"; - - private static final String FILE_ROOT_SUFFIX = "_FileRootTest"; - private static final String TXT_FILE = "FileContentTestFile.txt"; - - private Map _expectedPaths; - - @Test - public void fileRootsTest() - { - //pre-clean - cleanup(); - - _expectedPaths = new HashMap<>(); - - FileContentService svc = FileContentService.get(); - Assert.assertNotNull(svc); - - Container project1 = ContainerManager.createContainer(ContainerManager.getRoot(), PROJECT1, TestContext.get().getUser()); - _expectedPaths.put(project1, null); - - Container project2 = ContainerManager.createContainer(ContainerManager.getRoot(), PROJECT2, TestContext.get().getUser()); - _expectedPaths.put(project2, null); - - Container subfolder1 = ContainerManager.createContainer(project1, PROJECT1_SUBFOLDER1, TestContext.get().getUser()); - _expectedPaths.put(subfolder1, null); - - Container subfolder2 = ContainerManager.createContainer(project1, PROJECT1_SUBFOLDER2, TestContext.get().getUser()); - _expectedPaths.put(subfolder2, null); - - Container subsubfolder = ContainerManager.createContainer(subfolder1, PROJECT1_SUBSUBFOLDER, TestContext.get().getUser()); - _expectedPaths.put(subsubfolder, null); - - //set custom root on project, then expect children to inherit - File testRoot = getTestRoot(); - - svc.setFileRoot(project1, testRoot); - _expectedPaths.put(project1, testRoot); - - //the subfolder should inherit from the parent - _expectedPaths.put(subfolder1, new File(testRoot, subfolder1.getName())); - assertPathsEqual("Incorrect values returned by getDefaultRoot", _expectedPaths.get(subfolder1), svc.getDefaultRoot(subfolder1, false)); - assertPathsEqual("Subfolder1 has incorrect root", _expectedPaths.get(subfolder1), svc.getFileRoot(subfolder1)); - - _expectedPaths.put(subfolder2, new File(testRoot, subfolder2.getName())); - assertPathsEqual("Incorrect values returned by getDefaultRoot", _expectedPaths.get(subfolder2), svc.getDefaultRoot(subfolder2, false)); - assertPathsEqual("Subfolder2 has incorrect root", _expectedPaths.get(subfolder2), svc.getFileRoot(subfolder2)); - - _expectedPaths.put(subsubfolder, new File(_expectedPaths.get(subfolder1), subsubfolder.getName())); - assertPathsEqual("Incorrect values returned by getDefaultRoot", _expectedPaths.get(subsubfolder), svc.getDefaultRoot(subsubfolder, false)); - assertPathsEqual("SubSubfolder has incorrect root", _expectedPaths.get(subsubfolder), svc.getFileRoot(subsubfolder)); - - //override root on 1st child, expect children of that folder to inherit - _expectedPaths.put(subfolder1, new File(testRoot, "CustomSubfolder")); - _expectedPaths.get(subfolder1).mkdirs(); - svc.setFileRoot(subfolder1, _expectedPaths.get(subfolder1)); - assertPathsEqual("SubSubfolder has incorrect root", new File(_expectedPaths.get(subfolder1), subsubfolder.getName()), svc.getFileRoot(subsubfolder)); - - //reset project, we assume overridden child roots to remain the same - svc.setFileRoot(project1, null); - assertPathsEqual("Subfolder1 has incorrect root", _expectedPaths.get(subfolder1), svc.getFileRoot(subfolder1)); - assertPathsEqual("SubSubfolder has incorrect root", new File(_expectedPaths.get(subfolder1), subsubfolder.getName()), svc.getFileRoot(subsubfolder)); - - } - - private void assertPathsEqual(String msg, File expected, File actual) - { - String expectedPath = FileUtil.getAbsoluteCaseSensitiveFile(expected).getPath(); - String actualPath = FileUtil.getAbsoluteCaseSensitiveFile(actual).getPath(); - Assert.assertEquals(msg, expectedPath, actualPath); - } - - private File getTestRoot() - { - FileContentService svc = FileContentService.get(); - File siteRoot = svc.getSiteDefaultRoot(); - File testRoot = new File(siteRoot, FILE_ROOT_SUFFIX); - testRoot.mkdirs(); - Assert.assertTrue("Unable to create test file root", NetworkDrive.exists(testRoot)); - - return testRoot; - } - - @Test - //when we move a folder, we expect child files to follow, and expect - // any file paths stored in the DB to also get updated - public void testFolderMove() throws Exception - { - //pre-clean - cleanup(); - - _expectedPaths = new HashMap<>(); - - FileContentService svc = FileContentService.get(); - Assert.assertNotNull(svc); - - Container project1 = ContainerManager.createContainer(ContainerManager.getRoot(), PROJECT1, TestContext.get().getUser()); - _expectedPaths.put(project1, null); - - Container project2 = ContainerManager.createContainer(ContainerManager.getRoot(), PROJECT2, TestContext.get().getUser()); - _expectedPaths.put(project2, null); - - Container subfolder1 = ContainerManager.createContainer(project1, PROJECT1_SUBFOLDER1, TestContext.get().getUser()); - _expectedPaths.put(subfolder1, null); - - Container subfolder2 = ContainerManager.createContainer(project1, PROJECT1_SUBFOLDER2, TestContext.get().getUser()); - _expectedPaths.put(subfolder2, null); - - Container subsubfolder = ContainerManager.createContainer(subfolder1, PROJECT1_SUBSUBFOLDER, TestContext.get().getUser()); - Container subsubfolderSibling = ContainerManager.createContainer(subfolder1, PROJECT1_SUBSUBFOLDER_SIBLING, TestContext.get().getUser()); - _expectedPaths.put(subsubfolder, null); - - //create a test file that we will follow - File fileRoot = svc.getFileRoot(subsubfolder, ContentType.files); - fileRoot.mkdirs(); - - File childFile = new File(fileRoot, TXT_FILE); - childFile.createNewFile(); - - ExpData data = ExperimentService.get().createData(subsubfolder, UPLOADED_FILE); - data.setDataFileURI(childFile.toPath().toUri()); - data.save(TestContext.get().getUser()); - - ExpProtocol protocol = ExperimentService.get().createExpProtocol(subsubfolder, ExpProtocol.ApplicationType.ProtocolApplication, "DummyProtocol"); - protocol = ExperimentService.get().insertSimpleProtocol(protocol, TestContext.get().getUser()); - - ExpRun expRun = ExperimentService.get().createExperimentRun(subsubfolder, "DummyRun"); - expRun.setProtocol(protocol); - expRun.setFilePathRootPath(childFile.getParentFile().toPath()); - - ViewBackgroundInfo info = new ViewBackgroundInfo(subsubfolder, TestContext.get().getUser(), null); - ExpRun run = ExperimentService.get().saveSimpleExperimentRun( - expRun, - Collections.emptyMap(), - Collections.singletonMap(data, "Data"), - Collections.emptyMap(), - Collections.emptyMap(), - Collections.emptyMap(), - info, - _log, - false); - - Assert.assertTrue("File not found: " + childFile.getPath(), NetworkDrive.exists(childFile)); - ContainerManager.move(subsubfolder, subfolder2, TestContext.get().getUser()); - Container movedSubfolder = ContainerManager.getChild(subfolder2, subsubfolder.getName()); - - _expectedPaths.put(movedSubfolder, new File(svc.getFileRoot(subfolder2), movedSubfolder.getName())); - assertPathsEqual("Incorrect values returned by getDefaultRoot", _expectedPaths.get(movedSubfolder), svc.getDefaultRoot(movedSubfolder, false)); - assertPathsEqual("SubSubfolder has incorrect root", _expectedPaths.get(movedSubfolder), svc.getFileRoot(movedSubfolder)); - - File expectedFile = new File(svc.getFileRoot(movedSubfolder, ContentType.files), TXT_FILE); - Assert.assertTrue("File was not moved, expected: " + expectedFile.getPath(), NetworkDrive.exists(expectedFile)); - - ExpData movedData = ExperimentService.get().getExpData(data.getRowId()); - Assert.assertNotNull(movedData); - - // Reload the run after it's path has hopefully been updated - expRun = ExperimentService.get().getExpRun(expRun.getRowId()); - - assertPathsEqual("Incorrect data file path", expectedFile, FileUtil.stringToPath(movedSubfolder, movedData.getDataFileUrl()).toFile()); - assertPathsEqual("Incorrect run root path", expectedFile.getParentFile(), expRun.getFilePathRoot()); - - // Issue 38206 - file paths get mangled with multiple folder moves - ContainerManager.move(subsubfolderSibling, subfolder2, TestContext.get().getUser()); - - // Reload the run after it's path has hopefully NOT been updated - expRun = ExperimentService.get().getExpRun(expRun.getRowId()); - assertPathsEqual("Incorrect run root path", expectedFile.getParentFile(), expRun.getFilePathRoot()); - } - - @Test - public void testWorkbooksAndTabs() - { - //pre-clean - cleanup(); - - FileContentService svc = FileContentService.get(); - Assert.assertNotNull(svc); - - Container project1 = ContainerManager.createContainer(ContainerManager.getRoot(), PROJECT1, TestContext.get().getUser()); - - Container workbook = ContainerManager.createContainer(project1, null, null, null, WorkbookContainerType.NAME, TestContext.get().getUser()); - File expectedWorkbookRoot = new File(svc.getFileRoot(project1), workbook.getName()); - assertPathsEqual("Workbook has incorrect file root", expectedWorkbookRoot, svc.getFileRoot(workbook)); - - Container tab = ContainerManager.createContainer(project1, "tab", null, null, TabContainerType.NAME, TestContext.get().getUser()); - File expectedTabRoot = new File(svc.getFileRoot(project1), tab.getName()); - assertPathsEqual("Folder tab has incorrect file root", expectedTabRoot, svc.getFileRoot(tab)); - } - - /** - * Test that the Site Settings can be configured from startup properties - */ - @Test - public void testStartupPropertiesForSiteRootSettings() throws IOException - { - // save the original Site Root File settings so that we can restore them when this test is done - File originalSiteRootFile = FileContentService.get().getSiteDefaultRoot(); - - // create the new site root file to test with as a child of the current site root file so that we know it is in a dir that exist - String originalSiteRootFilePath = originalSiteRootFile.getAbsolutePath(); - File testSiteRootFile = new File(originalSiteRootFilePath, "testSiteRootFile"); - testSiteRootFile.createNewFile(); - - ModuleLoader.getInstance().handleStartupProperties(new RandomSiteSettingsPropertyHandler(){ - @Override - public @NotNull Collection getStartupPropertyEntries() - { - return List.of(new StartupPropertyEntry("siteFileRoot", testSiteRootFile.getAbsolutePath(), "startup", SCOPE_SITE_SETTINGS)); - } - - @Override - public boolean performChecks() - { - return false; - } - }); - - // now check that the expected changes occurred to the Site Root File settings on the server - File newSiteRootFile = FileContentService.get().getSiteDefaultRoot(); - Assert.assertEquals("The expected change in Site Root File was not found", testSiteRootFile.getAbsolutePath(), newSiteRootFile.getAbsolutePath()); - - // restore the Site Root File server settings to how they were originally - FileContentService.get().setSiteDefaultRoot(originalSiteRootFile, null); - testSiteRootFile.delete(); - } - - @After - public void cleanup() - { - FileContentService svc = FileContentService.get(); - Assert.assertNotNull(svc); - - deleteContainerAndFiles(svc, ContainerManager.getForPath(PROJECT1)); - deleteContainerAndFiles(svc, ContainerManager.getForPath(PROJECT2)); - - File testRoot = getTestRoot(); - if (NetworkDrive.exists(testRoot)) - { - FileUtil.deleteDir(testRoot); - } - } - - private void deleteContainerAndFiles(FileContentService svc, @Nullable Container c) - { - if (c != null) - { - ContainerManager.deleteAll(c, TestContext.get().getUser()); - - File file1 = svc.getFileRoot(c); - if (NetworkDrive.exists(file1)) - { - FileUtil.deleteDir(file1); - } - } - } - } -} +/* + * Copyright (c) 2009-2019 LabKey Corporation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.labkey.filecontent; + +import org.apache.commons.io.FilenameUtils; +import org.apache.commons.lang3.StringUtils; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.jetbrains.annotations.NotNull; +import org.jetbrains.annotations.Nullable; +import org.junit.After; +import org.junit.Assert; +import org.junit.Test; +import org.labkey.api.action.SpringActionController; +import org.labkey.api.admin.AdminUrls; +import org.labkey.api.attachments.AttachmentDirectory; +import org.labkey.api.cache.Cache; +import org.labkey.api.cache.CacheManager; +import org.labkey.api.cloud.CloudStoreService; +import org.labkey.api.collections.CaseInsensitiveHashMap; +import org.labkey.api.data.CompareType; +import org.labkey.api.data.Container; +import org.labkey.api.data.ContainerManager; +import org.labkey.api.data.ContainerManager.ContainerListener; +import org.labkey.api.data.ContainerType; +import org.labkey.api.data.CoreSchema; +import org.labkey.api.data.SQLFragment; +import org.labkey.api.data.SimpleFilter; +import org.labkey.api.data.TabContainerType; +import org.labkey.api.data.Table; +import org.labkey.api.data.TableInfo; +import org.labkey.api.data.TableSelector; +import org.labkey.api.data.WorkbookContainerType; +import org.labkey.api.exp.Lsid; +import org.labkey.api.exp.api.ExpData; +import org.labkey.api.exp.api.ExpProtocol; +import org.labkey.api.exp.api.ExpRun; +import org.labkey.api.exp.api.ExperimentService; +import org.labkey.api.exp.query.ExpDataTable; +import org.labkey.api.files.DirectoryPattern; +import org.labkey.api.files.FileContentService; +import org.labkey.api.files.FileListener; +import org.labkey.api.files.FileRoot; +import org.labkey.api.files.FilesAdminOptions; +import org.labkey.api.files.MissingRootDirectoryException; +import org.labkey.api.files.UnsetRootDirectoryException; +import org.labkey.api.files.view.FilesWebPart; +import org.labkey.api.module.Module; +import org.labkey.api.module.ModuleLoader; +import org.labkey.api.pipeline.PipeRoot; +import org.labkey.api.pipeline.PipelineService; +import org.labkey.api.pipeline.PipelineUrls; +import org.labkey.api.query.BatchValidationException; +import org.labkey.api.query.FieldKey; +import org.labkey.api.query.QueryUpdateService; +import org.labkey.api.security.User; +import org.labkey.api.security.permissions.AdminOperationsPermission; +import org.labkey.api.settings.AppProps; +import org.labkey.api.settings.RandomSiteSettingsPropertyHandler; +import org.labkey.api.settings.StartupPropertyEntry; +import org.labkey.api.settings.WriteableAppProps; +import org.labkey.api.test.TestWhen; +import org.labkey.api.util.ContainerUtil; +import org.labkey.api.util.DOM; +import org.labkey.api.util.FileUtil; +import org.labkey.api.util.GUID; +import org.labkey.api.util.HtmlString; +import org.labkey.api.util.NetworkDrive; +import org.labkey.api.util.PageFlowUtil; +import org.labkey.api.util.Path; +import org.labkey.api.util.TestContext; +import org.labkey.api.util.URIUtil; +import org.labkey.api.view.ActionURL; +import org.labkey.api.view.HttpView; +import org.labkey.api.view.ViewBackgroundInfo; +import org.labkey.api.view.ViewContext; +import org.labkey.api.view.template.WarningProvider; +import org.labkey.api.view.template.WarningService; +import org.labkey.api.view.template.Warnings; +import org.labkey.api.webdav.WebdavResource; +import org.labkey.api.webdav.WebdavService; + +import java.beans.PropertyChangeEvent; +import java.io.BufferedWriter; +import java.io.File; +import java.io.IOException; +import java.net.URI; +import java.net.URISyntaxException; +import java.nio.file.Files; +import java.nio.file.InvalidPathException; +import java.nio.file.StandardOpenOption; +import java.util.ArrayList; +import java.util.Collection; +import java.util.Collections; +import java.util.Date; +import java.util.HashMap; +import java.util.HashSet; +import java.util.LinkedHashMap; +import java.util.LinkedHashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.concurrent.CopyOnWriteArrayList; +import java.util.regex.Pattern; +import java.util.stream.Stream; + +import static org.labkey.api.settings.AppProps.SCOPE_SITE_SETTINGS; +import static org.labkey.api.util.DOM.Attribute.href; +import static org.labkey.api.util.DOM.at; + +public class FileContentServiceImpl implements FileContentService, WarningProvider +{ + private static final Logger _log = LogManager.getLogger(FileContentServiceImpl.class); + private static final String UPLOAD_LOG = ".upload.log"; + private static final FileContentServiceImpl INSTANCE = new FileContentServiceImpl(); + + private final ContainerListener _containerListener = new FileContentServiceContainerListener(); + private final List _fileListeners = new CopyOnWriteArrayList<>(); + + private final List _ziploaderPattern = new CopyOnWriteArrayList<>(); + + private volatile boolean _fileRootSetViaStartupProperty = false; + private String _problematicFileRootMessage; + + enum FileAction + { + UPLOAD, + DELETE + } + + static FileContentServiceImpl getInstance() + { + return INSTANCE; + } + + private FileContentServiceImpl() + { + WarningService.get().register(this); + } + + @Override + @NotNull + public List getContainersForFilePath(java.nio.file.Path path) + { + // Ignore cloud files for now + if (FileUtil.hasCloudScheme(path)) + return Collections.emptyList(); + + // If the path is under the default root, do optimistic simple match for containers under the default root + File defaultRoot = getSiteDefaultRoot(); + java.nio.file.Path defaultRootPath = defaultRoot.toPath(); + if (path.startsWith(defaultRootPath)) + { + java.nio.file.Path rel = defaultRootPath.relativize(path); + if (rel.getNameCount() > 0) + { + Container root = ContainerManager.getRoot(); + Container next = root; + while (rel.getNameCount() > 0) + { + // check if there exists a child container that matches the next path segment + java.nio.file.Path top = rel.subpath(0, 1); + assert top != null; + Container child = next.getChild(top.getFileName().toString()); + if (child == null) + break; + + next = child; + + if(rel.getNameCount() > 1) + { + rel = rel.subpath(1, rel.getNameCount()); + } + else + { + break; + } + } + + if (next != null && !next.equals(root)) + { + // verify our naive file path is correct for the container -- it may have a file root other than the default + java.nio.file.Path fileRoot = getFileRootPath(next); + if (fileRoot != null && path.startsWith(fileRoot)) + return Collections.singletonList(next); + } + } + } + + // TODO: Create cache of file root and pipeline root paths -> list of containers + + return Collections.emptyList(); + } + + @Override + public @Nullable File getFileRoot(@NotNull Container c, @NotNull ContentType type) + { + switch (type) + { + case files: + case assayfiles: + String folderName = getFolderName(type); + if (folderName == null) + folderName = ""; + + java.nio.file.Path dir = getFileRootPath(c); + return dir != null ? dir.resolve(folderName).toFile() : null; + + case pipeline: + PipeRoot root = PipelineService.get().findPipelineRoot(c); + return root != null ? root.getRootPath() : null; + } + return null; + } + + @Override + public @Nullable java.nio.file.Path getFileRootPath(@NotNull Container c, @NotNull ContentType type) + { + switch (type) + { + case files: + case assayfiles: + java.nio.file.Path fileRootPath = getFileRootPath(c); + if (null != fileRootPath && !FileUtil.hasCloudScheme(fileRootPath)) // Don't add @files when we're in the cloud + fileRootPath = fileRootPath.resolve(getFolderName(type)); + return fileRootPath; + + case pipeline: + PipeRoot root = PipelineService.get().findPipelineRoot(c); + return root != null ? root.getRootNioPath() : null; + } + return null; + } + + // Returns full uri to file root for this container. filePath is optional relative path to a file under the file root + @Override + public @Nullable URI getFileRootUri(@NotNull Container c, @NotNull ContentType type, @Nullable String filePath) + { + java.nio.file.Path root = FileContentService.get().getFileRootPath(c, FileContentService.ContentType.files); + if (root != null) + { + String path = root.toString(); + if (filePath != null) { + path += filePath; + } + + // non-unix needs a leading slash + if (!path.startsWith("/") && !path.startsWith("\\")) + { + path = "/" + path; + } + return FileUtil.createUri(path); + } + + return null; + } + + @Override + public @Nullable File getFileRoot(@NotNull Container c) + { + java.nio.file.Path path = getFileRootPath(c); + throwIfPathNotFile(path, c); + return path.toFile(); + } + + @Override + public @Nullable java.nio.file.Path getFileRootPath(@NotNull Container c) + { + if (c == null) + return null; + + if (c.isRoot()) + { + return getSiteDefaultRootPath(); + } + + if (!isFileRootDisabled(c)) + { + FileRoot root = FileRootManager.get().getFileRoot(c); + + // check if there is a site wide file root + if (root.getPath() == null || isUseDefaultRoot(c)) + { + return getDefaultRootPath(c, true); + } + else + return getNioPath(c, root.getPath()); + } + return null; + } + + @Override + public File getDefaultRoot(Container c, boolean createDir) + { + return getDefaultRootPath(c, createDir).toFile(); + } + + @Override + public java.nio.file.Path getDefaultRootPath(@NotNull Container c, boolean createDir) + { + Container firstOverride = getFirstAncestorWithOverride(c); + + java.nio.file.Path parentRoot; + if (firstOverride == null) + { + parentRoot = getSiteDefaultRoot().toPath(); + firstOverride = ContainerManager.getRoot(); + } + else + { + parentRoot = getFileRootPath(firstOverride); + } + + if (parentRoot != null && firstOverride != null) + { + java.nio.file.Path fileRootPath; + if (FileUtil.hasCloudScheme(parentRoot)) + { + // For cloud root, we don't have to create directories for this path + fileRootPath = CloudStoreService.get().getPathForOtherContainer(firstOverride, c, FileUtil.pathToString(parentRoot), new Path("")); + } + else + { + // For local, the path may be several directories deep (since it matches the LK folder path), so we should create the directories for that path + fileRootPath = FileUtil.appendPath(parentRoot.toFile(), Path.parse(getRelativePath(c, firstOverride))).toPath(); + + try + { + if (createDir && !NetworkDrive.exists(fileRootPath)) + FileUtil.createDirectories(fileRootPath); + } + catch (IOException e) + { + return null; // throw new RuntimeException(e); TODO: does returning null make certain tests, like TargetedMSQCGuideSetTest pass on Windows? + } + } + + return fileRootPath; + } + return null; + } + + // Return pretty path string for defaultFileRoot and boolean true if defaultFileRoot is cloud + @Override + public DefaultRootInfo getDefaultRootInfo(Container container) + { + String defaultRoot = ""; + boolean isDefaultRootCloud = false; + java.nio.file.Path defaultRootPath = getDefaultRootPath(container, false); + String cloudName = null; + if (defaultRootPath != null) + { + isDefaultRootCloud = FileUtil.hasCloudScheme(defaultRootPath); + if (isDefaultRootCloud && !container.isProject()) + { + FileRoot fileRoot = getDefaultFileRoot(container); + if (null != fileRoot) + defaultRoot = fileRoot.getPath(); + if (null != defaultRoot) + cloudName = getCloudRootName(defaultRoot); + } + else + { + defaultRoot = FileUtil.getAbsolutePath(container, defaultRootPath.toUri()); + } + } + return new DefaultRootInfo(defaultRootPath, defaultRoot, isDefaultRootCloud, cloudName); + } + + @Nullable + // Get FileRoot associated with path returned form getDefaultRootPath() + public FileRoot getDefaultFileRoot(Container c) + { + Container firstOverride = getFirstAncestorWithOverride(c); + + if (firstOverride == null) + firstOverride = ContainerManager.getRoot(); + + if (null != firstOverride) + return FileRootManager.get().getFileRoot(firstOverride); + return null; + } + + private @NotNull String getRelativePath(Container c, Container ancestor) + { + return c.getPath().replaceAll("^" + Pattern.quote(ancestor.getPath()), ""); + } + + //returns the first parent container that has a custom file root, or NULL if none have overrides + private Container getFirstAncestorWithOverride(Container c) + { + Container toTest = c.getParent(); + if (toTest == null) + return null; + + while (isUseDefaultRoot(toTest)) + { + if (toTest == null || toTest.equals(ContainerManager.getRoot())) + return null; + + toTest = toTest.getParent(); + } + + return toTest; + } + + private java.nio.file.Path getNioPath(Container c, @NotNull String fileRootPath) + { + if (isCloudFileRoot(fileRootPath)) + return CloudStoreService.get().getPath(c, getCloudRootName(fileRootPath), new org.labkey.api.util.Path("")); + + return FileUtil.stringToPath(c, fileRootPath, false); // fileRootPath is unencoded + } + + private boolean isCloudFileRoot(String fileRootPseudoPath) + { + return StringUtils.startsWith(fileRootPseudoPath, FileContentService.CLOUD_ROOT_PREFIX); + } + + private String getCloudRootName(@NotNull String fileRootPseudoPath) + { + return fileRootPseudoPath.substring(fileRootPseudoPath.indexOf(FileContentService.CLOUD_ROOT_PREFIX) + FileContentService.CLOUD_ROOT_PREFIX.length() + 1); + } + + @Override + public boolean isCloudRoot(Container c) + { + if (null != c) + { + java.nio.file.Path fileRootPath = getFileRootPath(c); + return null != fileRootPath && FileUtil.hasCloudScheme(fileRootPath); + } + return false; + } + + @Override + @NotNull + public String getCloudRootName(Container c) + { + if (null != c) + { + if (isCloudRoot(c)) + { + FileRoot root = FileRootManager.get().getFileRoot(c); + if (null == root.getPath() || isUseDefaultRoot(c)) + { + Container firstOverride = getFirstAncestorWithOverride(c); + if (null == firstOverride) + firstOverride = ContainerManager.getRoot(); + root = FileRootManager.get().getFileRoot(firstOverride); + if (null == root.getPath()) + return ""; + } + return getCloudRootName(root.getPath()); + } + } + return ""; + } + + @Override + public void setCloudRoot(@NotNull Container c, String cloudRootName) + { + _setFileRoot(c, FileContentService.CLOUD_ROOT_PREFIX + "/" + cloudRootName); + } + + @Override + public void setFileRoot(@NotNull Container c, @Nullable File path) + { + _setFileRoot(c, (null != path ? FileUtil.getAbsoluteCaseSensitiveFile(path).getAbsolutePath() : null)); + } + + @Override + public void setFileRootPath(@NotNull Container c, @Nullable String strPath) + { + String absolutePath = null; + if (strPath != null) + { + URI uri = FileUtil.createUri(strPath, false); // strPath is unencoded + if (FileUtil.hasCloudScheme(uri)) + absolutePath = FileUtil.getAbsolutePath(c, uri); + else + absolutePath = FileUtil.getAbsoluteCaseSensitiveFile(new File(uri)).getAbsolutePath(); + } + _setFileRoot(c, absolutePath); + } + + private void _setFileRoot(@NotNull Container c, @Nullable String absolutePath) + { + if (!c.isContainerFor(ContainerType.DataType.fileRoot)) + throw new IllegalArgumentException("File roots cannot be set for containers of type " + c.getContainerType().getName()); + + FileRoot root = FileRootManager.get().getFileRoot(c); + root.setEnabled(true); + + String oldValue = root.getPath(); + String newValue = null; + + // clear out the root + if (absolutePath == null) + root.setPath(null); + else + { + root.setPath(absolutePath); + newValue = root.getPath(); + } + + FileRootManager.get().saveFileRoot(null, root); + ContainerManager.ContainerPropertyChangeEvent evt = new ContainerManager.ContainerPropertyChangeEvent( + c, ContainerManager.Property.WebRoot, oldValue, newValue); + ContainerManager.firePropertyChangeEvent(evt); + } + + @Override + public void disableFileRoot(Container container) + { + if (container == null || container.isRoot()) + throw new IllegalArgumentException("Disabling either a null project or the root project is not allowed."); + + Container effective = container.getContainerFor(ContainerType.DataType.fileRoot); + if (effective != null) + { + FileRoot root = FileRootManager.get().getFileRoot(effective); + String oldValue = root.getPath(); + root.setEnabled(false); + FileRootManager.get().saveFileRoot(null, root); + + ContainerManager.ContainerPropertyChangeEvent evt = new ContainerManager.ContainerPropertyChangeEvent( + container, ContainerManager.Property.WebRoot, oldValue, null); + ContainerManager.firePropertyChangeEvent(evt); + } + } + + @Override + public boolean isFileRootDisabled(Container c) + { + Container effective = c.getContainerFor(ContainerType.DataType.fileRoot); + if (null == effective) + return false; + + FileRoot root = FileRootManager.get().getFileRoot(effective); + return !root.isEnabled(); + } + + @Override + public boolean isUseDefaultRoot(Container c) + { + if (c == null) + return true; + + Container effective = c.getContainerFor(ContainerType.DataType.fileRoot); + if (null == effective) + return true; + + FileRoot root = FileRootManager.get().getFileRoot(effective); + return root.isUseDefault() || StringUtils.isEmpty(root.getPath()); + } + + @Override + public void setIsUseDefaultRoot(Container c, boolean useDefaultRoot) + { + Container effective = c.getContainerFor(ContainerType.DataType.fileRoot); + if (effective != null) + { + FileRoot root = FileRootManager.get().getFileRoot(effective); + String oldValue = root.getPath(); + root.setEnabled(true); + root.setUseDefault(useDefaultRoot); + if (useDefaultRoot) + root.setPath(null); + FileRootManager.get().saveFileRoot(null, root); + + ContainerManager.ContainerPropertyChangeEvent evt = new ContainerManager.ContainerPropertyChangeEvent( + effective, ContainerManager.Property.WebRoot, oldValue, null); + ContainerManager.firePropertyChangeEvent(evt); + } + } + + @Override + public @NotNull java.nio.file.Path getSiteDefaultRootPath() + { + return getSiteDefaultRoot().toPath(); + } + + @Override + public @NotNull File getSiteDefaultRoot() + { + // Site default is always on file system + File root = AppProps.getInstance().getFileSystemRoot(); + + try + { + if (!NetworkDrive.exists(root)) + { + File configuredRoot = root; + root = getDefaultRoot(); + if (configuredRoot != null && !configuredRoot.equals(root)) + { + String message = "The configured site-wide file root " + configuredRoot + " does not exist. Falling back to " + root; + if (!message.equals(_problematicFileRootMessage)) + { + _problematicFileRootMessage = message; + _log.error(_problematicFileRootMessage); + } + } + } + else + { + _problematicFileRootMessage = null; + } + + if (!NetworkDrive.exists(root)) + { + if (FileUtil.mkdirs(root)) + { + _log.info("Created site-wide file root " + root); + } + else + { + _log.error("Failed when attempting to create site-wide file root " + root); + } + } + } + catch (IOException e) + { + throw new RuntimeException("Unable to create file root directory", e); + } + + return root; + } + + @Override + public String getProblematicFileRootMessage() + { + return _problematicFileRootMessage; + } + + private @NotNull File getDefaultRoot() throws IOException + { + File explodedPath = ModuleLoader.getInstance().getCoreModule().getExplodedPath(); + + File root = explodedPath.getParentFile(); + if (root != null) + { + if (root.getParentFile() != null) + root = root.getParentFile(); + } + File defaultRoot = new File(root, "files"); + if (!NetworkDrive.exists(defaultRoot)) + FileUtil.mkdirs(defaultRoot); + + return defaultRoot; + } + + @Override + public void setSiteDefaultRoot(File root, User user) + { + if (root == null) + throw new IllegalArgumentException("Invalid site root: specified root is null"); + + if (!NetworkDrive.exists(root)) + throw new IllegalArgumentException("Invalid site root: " + root.getAbsolutePath() + " does not exist"); + + File prevRoot = getSiteDefaultRoot(); + WriteableAppProps props = AppProps.getWriteableInstance(); + + props.setFileSystemRoot(root.getAbsolutePath()); + props.save(user); + + FileRootManager.get().clearCache(); + ContainerManager.ContainerPropertyChangeEvent evt = new ContainerManager.ContainerPropertyChangeEvent( + ContainerManager.getRoot(), ContainerManager.Property.SiteRoot, prevRoot, root); + ContainerManager.firePropertyChangeEvent(evt); + } + + @Override + public void setWebfilesEnabled(boolean enabled, User user) + { + WriteableAppProps props = AppProps.getWriteableInstance(); + props.setWebfilesEnabled(enabled); + props.save(user); + } + + @Override + public FileSystemAttachmentParent registerDirectory(Container c, String name, String path, boolean relative) + { + FileSystemAttachmentParent parent = new FileSystemAttachmentParent(); + parent.setContainer(c); + if (null == name) + name = path; + parent.setName(name); + parent.setPath(path); + parent.setRelative(relative); + //We do this because insert does not return new fields + parent.setEntityid(GUID.makeGUID()); + + FileSystemAttachmentParent ret = Table.insert(HttpView.currentContext().getUser(), CoreSchema.getInstance().getMappedDirectories(), parent); + ContainerManager.ContainerPropertyChangeEvent evt = new ContainerManager.ContainerPropertyChangeEvent( + c, ContainerManager.Property.AttachmentDirectory, null, ret); + ContainerManager.firePropertyChangeEvent(evt); + return ret; + } + + @Override + public void unregisterDirectory(Container c, String name) + { + FileSystemAttachmentParent parent = getRegisteredDirectory(c, name); + SimpleFilter filter = SimpleFilter.createContainerFilter(c); + filter.addCondition(FieldKey.fromParts("Name"), name); + Table.delete(CoreSchema.getInstance().getMappedDirectories(), filter); + ContainerManager.ContainerPropertyChangeEvent evt = new ContainerManager.ContainerPropertyChangeEvent( + c, ContainerManager.Property.AttachmentDirectory, parent, null); + ContainerManager.firePropertyChangeEvent(evt); + } + + @Override + public @Nullable AttachmentDirectory getMappedAttachmentDirectory(Container c, boolean createDir) throws UnsetRootDirectoryException, MissingRootDirectoryException + { + return getMappedAttachmentDirectory(c, ContentType.files, createDir); + } + + @Override + @Nullable + public AttachmentDirectory getMappedAttachmentDirectory(Container c, ContentType contentType, boolean createDir) throws UnsetRootDirectoryException + { + try + { + if (createDir) //force create + getMappedDirectory(c, true); + else if (null == getMappedDirectory(c, false)) + return null; + + return new FileSystemAttachmentParent(c, contentType); + } + catch (IOException e) + { + _log.error("Cannot get mapped directory for " + c.getPath(), e); + return null; + } + } + + public java.nio.file.Path getMappedDirectory(Container c, boolean create) throws UnsetRootDirectoryException, IOException + { + java.nio.file.Path root = getFileRootPath(c); + if (!FileUtil.hasCloudScheme(root)) + { + if (null == root) + { + if (create) + throw new UnsetRootDirectoryException(c.isRoot() ? c : c.getProject()); + else + return null; + } + + if (!NetworkDrive.exists(root)) + { + if (create) + throw new MissingRootDirectoryException(c.isRoot() ? c : c.getProject(), root); + else + return null; + + } + } + return root; + } + + @Override + public FileSystemAttachmentParent getRegisteredDirectory(Container c, String name) + { + SimpleFilter filter = SimpleFilter.createContainerFilter(c); + filter.addCondition(FieldKey.fromParts("Name"), name); + + return new TableSelector(CoreSchema.getInstance().getMappedDirectories(), filter, null).getObject(FileSystemAttachmentParent.class); + } + + @Override + public FileSystemAttachmentParent getRegisteredDirectoryFromEntityId(Container c, String entityId) + { + SimpleFilter filter = SimpleFilter.createContainerFilter(c); + filter.addCondition(FieldKey.fromParts("EntityId"), entityId); + + return new TableSelector(CoreSchema.getInstance().getMappedDirectories(), filter, null).getObject(FileSystemAttachmentParent.class); + } + + @Override + public @NotNull Collection getRegisteredDirectories(Container c) + { + SimpleFilter filter = SimpleFilter.createContainerFilter(c); + + return Collections.unmodifiableCollection(new TableSelector(CoreSchema.getInstance().getMappedDirectories(), filter, null).getCollection(FileSystemAttachmentParent.class)); + } + + private class FileContentServiceContainerListener implements ContainerListener + { + @Override + public void containerCreated(Container c, User user) + { + try + { + // Will create directory if it's a default dir + getMappedDirectory(c, false); + } + catch (IOException ex) + { + /* */ + } + } + + @Override + public void containerDeleted(Container c, User user) + { + java.nio.file.Path dir = null; + try + { + // don't delete the file contents if they have a project override + if (isUseDefaultRoot(c) && !isCloudRoot(c)) // Don't do anything for cloud root here. CloudContainerListener will handle + dir = getMappedDirectory(c, false); + + if (null != dir) + { + FileUtil.deleteDir(dir); + } + } + catch (Exception e) + { + _log.error("containerDeleted", e); + } + + ContainerUtil.purgeTable(CoreSchema.getInstance().getMappedDirectories(), c, null); + } + + @Override + public void containerMoved(Container c, Container oldParent, User user) + { + /* **** Cases: + SRC DEST + specific local path same -- no work + specific cloud path same -- no work + local default local default -- move tree + local default cloud default -- move tree + cloud default local default -- move tree + cloud default cloud default -- if change bucket, move tree + *************************************************************/ + if (isUseDefaultRoot(c)) + { + java.nio.file.Path srcParent = getFileRootPath(oldParent); + java.nio.file.Path dest = getFileRootPath(c); + if (null != srcParent && null != dest) + { + if (!FileUtil.hasCloudScheme(srcParent)) + { + File src = new File(srcParent.toFile(), c.getName()); + if (NetworkDrive.exists(src)) + { + if (!FileUtil.hasCloudScheme(dest)) + { + // local -> local + moveFileRoot(src, dest.toFile(), user, c); + } + else + { + // local -> cloud; source starts under @files + File filesSrc = FileUtil.appendName(src, FILES_LINK); + if (NetworkDrive.exists(filesSrc)) + moveFileRoot(filesSrc.toPath(), dest, user, c); + FileUtil.deleteDir(src); // moveFileRoot will delete @files, but we need to delete its parent + } + } + } + else + { + // Get source path using moving container and parent's config (cloudRoot), because that config must be the source config + java.nio.file.Path src = CloudStoreService.get().getPath(c, getCloudRootName(oldParent), new Path("")); + if (!FileUtil.hasCloudScheme(dest)) + { + // cloud -> local; destination is under @files + dest = dest.resolve(FILES_LINK); + moveFileRoot(src, dest, user, c); + } + else + { + // cloud -> cloud + if (!getCloudRootName(oldParent).equals(getCloudRootName(c))) + { + // Different configs + moveFileRoot(src, dest, user, c); + } + } + } + } + } + } + + @Override + public void propertyChange(PropertyChangeEvent propertyChangeEvent) + { + ContainerManager.ContainerPropertyChangeEvent evt = (ContainerManager.ContainerPropertyChangeEvent)propertyChangeEvent; + Container c = evt.container; + + switch (evt.property) + { + case Name: // container rename event + { + String oldValue = (String) propertyChangeEvent.getOldValue(); + String newValue = (String) propertyChangeEvent.getNewValue(); + + java.nio.file.Path location; + try + { + location = getMappedDirectory(c, false); + if (location != null && !FileUtil.hasCloudScheme(location)) // If cloud, folder name for container not dependent on Name + { + //Don't rely on container object. Seems not to point to the + //new location even AFTER rename. Just construct new file paths + File locationFile = location.toFile(); + File parentDir = locationFile.getParentFile(); + File oldLocation = new File(parentDir, oldValue); + File newLocation = new File(parentDir, newValue); + if (NetworkDrive.exists(newLocation)) + moveToDeleted(newLocation); + + if (NetworkDrive.exists(oldLocation)) + { + oldLocation.renameTo(newLocation); + fireFileMoveEvent(oldLocation, newLocation, evt.user, evt.container); + } + } + } + catch (IOException ex) + { + _log.error(ex); + } + + break; + } + } + } + } + + + @Override + public @Nullable String getFolderName(FileContentService.ContentType type) + { + if (type != null) + return "@" + type.name(); + return null; + } + + + /** + * Move the file or directory into a ".deleted" directory under the parent directory. + * @return True if successfully moved. + */ + private static boolean moveToDeleted(File fileToMove) throws IOException + { + if (!NetworkDrive.exists(fileToMove)) + return false; + + File parent = fileToMove.getParentFile(); + + File deletedDir = new File(parent, ".deleted"); + if (!NetworkDrive.exists(deletedDir)) + if (!FileUtil.mkdir(deletedDir)) + return false; + + File newLocation = new File(deletedDir, fileToMove.getName()); + if (NetworkDrive.exists(newLocation)) + FileUtil.deleteDir(newLocation); + + return fileToMove.renameTo(newLocation); + } + + static void logFileAction(java.nio.file.Path directory, String fileName, FileAction action, User user) + { + try (BufferedWriter fw = Files.newBufferedWriter(directory.resolve(UPLOAD_LOG), StandardOpenOption.APPEND, StandardOpenOption.CREATE)) + { + fw.write(action.toString() + "\t" + fileName + "\t" + new Date() + "\t" + (user == null ? "(unknown)" : user.getEmail()) + "\n"); + } + catch (Exception x) + { + //Just log it. + _log.error(x); + } + } + + @Override + public FilesAdminOptions getAdminOptions(Container c) + { + FileRoot root = FileRootManager.get().getFileRoot(c); + String xml = null; + + if (!StringUtils.isBlank(root.getProperties())) + { + xml = root.getProperties(); + } + return new FilesAdminOptions(c, xml); + } + + @Override + public void setAdminOptions(Container c, FilesAdminOptions options) + { + if (options != null) + { + setAdminOptions(c, options.serialize()); + } + } + + @Override + public void setAdminOptions(Container c, String properties) + { + FileRoot root = FileRootManager.get().getFileRoot(c); + + root.setProperties(properties); + FileRootManager.get().saveFileRoot(null, root); + } + + public static final String NAMESPACE_PREFIX = "FileProperties"; + public static final String PROPERTIES_DOMAIN = "File Properties"; + public static final String TYPE_PROPERTIES = "FileProperties"; + + @Override + public String getDomainURI(Container container) + { + return getDomainURI(container, getAdminOptions(container).getFileConfig()); + } + + @Override + public String getDomainURI(Container container, FilesAdminOptions.fileConfig config) + { + while (config == FilesAdminOptions.fileConfig.useParent && container != container.getParent()) + { + container = container.getParent(); + config = getAdminOptions(container).getFileConfig(); + } + + //String typeURI = "urn:lsid:" + AppProps.getInstance().getDefaultLsidAuthority() + ":List" + ".Folder-" + container.getRowId() + ":" + name; + + return new Lsid("urn:lsid:labkey.com:" + NAMESPACE_PREFIX + ".Folder-" + container.getRowId() + ':' + TYPE_PROPERTIES).toString(); + } + + @Override @Nullable + public ExpData getDataObject(WebdavResource resource, Container c) + { + return getDataObject(resource, c, null, false); + } + + @Nullable + private static ExpData getDataObject(WebdavResource resource, Container c, User user, boolean create) + { + // TODO: S3: seems to only be called from Search and currently we're not searching in cloud. SaveCustomPropsAction seems unused + if (resource != null) + { + File file = resource.getFile(); + if (file != null) + { + ExpData data = ExperimentService.get().getExpDataByURL(file, c); + + if (data == null && create) + { + data = ExperimentService.get().createData(c, FileContentService.UPLOADED_FILE); + data.setName(file.getName()); + data.setDataFileURI(file.toURI()); + data.save(user); + } + return data; + } + } + return null; + } + + @Override + public QueryUpdateService getFilePropsUpdateService(TableInfo tinfo, Container container) + { + return new FileQueryUpdateService(tinfo, container); + } + + @Override + public boolean isValidProjectRoot(String root) + { + File f = new File(root); + return NetworkDrive.exists(f) && f.isDirectory(); + } + + @Override + public void moveFileRoot(java.nio.file.Path prev, java.nio.file.Path dest, @Nullable User user, @Nullable Container container) + { + if (!FileUtil.hasCloudScheme(prev) && !FileUtil.hasCloudScheme(dest)) + { + moveFileRoot(prev.toFile(), dest.toFile(), user, container); // Both files; try rename + } + else + { + try + { + // At least one is in the cloud + FileUtil.copyDirectory(prev, dest); + FileUtil.deleteDir(prev); // TODO use more efficient delete + fireFileMoveEvent(prev, dest, user, container); + } + catch (IOException e) + { + _log.error("error occurred moving the file root", e); + } + } + } + + @Override + public void moveFileRoot(File prev, File dest, @Nullable User user, @Nullable Container container) + { + try + { + _log.info("moving " + prev.getPath() + " to " + dest.getPath()); + boolean doRename = true; + + // Our best bet for perf is to do a rename, which doesn't require creating an actual copy. + // If it exists, try deleting the target directory, which will only succeed if it's empty, but would + // enable using renameTo() method. Don't delete if it's a symbolic link, since it wouldn't be recreated + // in the same way. + if (NetworkDrive.exists(dest) && !Files.isSymbolicLink(dest.toPath())) + doRename = dest.delete(); + + if (doRename && !prev.renameTo(dest)) + { + _log.info("rename failed, attempting to copy"); + + //listFiles can return null, which could cause a NPE + File[] children = prev.listFiles(); + if (children != null) + { + for (File file : children) + FileUtil.copyBranch(file, dest); + } + FileUtil.deleteDir(prev); + } + fireFileMoveEvent(prev, dest, user, container); + } + catch (IOException e) + { + _log.error("error occurred moving the file root", e); + } + } + + @Override + public void fireFileCreateEvent(@NotNull File created, @Nullable User user, @Nullable Container container) + { + fireFileCreateEvent(created.toPath(), user, container); + } + + @Override + public void fireFileCreateEvent(@NotNull java.nio.file.Path created, @Nullable User user, @Nullable Container container) + { + java.nio.file.Path absPath = FileUtil.getAbsoluteCaseSensitivePath(container, created); + for (FileListener fileListener : _fileListeners) + { + fileListener.fileCreated(absPath, user, container); + } + } + + @Override + public void fireFileReplacedEvent(@NotNull java.nio.file.Path replaced, @Nullable User user, @Nullable Container container) + { + java.nio.file.Path absPath = FileUtil.getAbsoluteCaseSensitivePath(container, replaced); + for (FileListener fileListener : _fileListeners) + { + fileListener.fileReplaced(absPath, user, container); + } + } + + @Override + public void fireFileDeletedEvent(@NotNull java.nio.file.Path deleted, @Nullable User user, @Nullable Container container) + { + java.nio.file.Path absPath = FileUtil.getAbsoluteCaseSensitivePath(container, deleted); + for (FileListener fileListener : _fileListeners) + { + fileListener.fileDeleted(absPath, user, container); + } + } + + @Override + public int fireFileMoveEvent(@NotNull File src, @NotNull File dest, @Nullable User user, @Nullable Container container) + { + return fireFileMoveEvent(src.toPath(), dest.toPath(), user, container); + } + + @Override + public int fireFileMoveEvent(@NotNull java.nio.file.Path src, @NotNull java.nio.file.Path dest, @Nullable User user, @Nullable Container container) + { + return fireFileMoveEvent(src, dest, user, container, null); + } + + @Override + public int fireFileMoveEvent(@NotNull java.nio.file.Path src, @NotNull java.nio.file.Path dest, @Nullable User user, @Nullable Container sourceContainer, @Nullable Container targetContainer) + { + // Make sure that we've got the best representation of the file that we can + java.nio.file.Path absSrc = FileUtil.getAbsoluteCaseSensitivePath(sourceContainer, src); + java.nio.file.Path absDest = FileUtil.getAbsoluteCaseSensitivePath(targetContainer != null ? targetContainer : sourceContainer, dest); + int result = 0; + for (FileListener fileListener : _fileListeners) + { + result += fileListener.fileMoved(absSrc, absDest, user, sourceContainer, targetContainer); + } + return result; + } + + @Override + public void addFileListener(FileListener listener) + { + _fileListeners.add(listener); + } + + @Override + public Map> listFiles(@NotNull Container container) + { + Map> files = new LinkedHashMap<>(); + for (FileListener fileListener : _fileListeners) + { + files.put(fileListener.getSourceName(), new HashSet<>(fileListener.listFiles(container))); + } + return files; + } + + @Override + public SQLFragment listSampleFilesQuery(@NotNull User currentUser) + { + SQLFragment frag = new SQLFragment(); + String union = ""; + frag.append("("); + + for (FileListener fileListener : _fileListeners) + { + SQLFragment subselect = fileListener.listSampleFilesQuery(); + if (subselect != null) + { + frag.append(union); + frag.append(subselect); + union = "UNION\n"; + } + } + frag.append(")"); + return frag; + } + + @Override + public SQLFragment listFilesQuery(@NotNull User currentUser) + { + SQLFragment frag = new SQLFragment(); + if (currentUser == null || !currentUser.hasSiteAdminPermission()) + { + frag.append("SELECT\n"); + frag.append(" CAST(NULL AS VARCHAR) AS Container,\n"); + frag.append(" NULL AS Created,\n"); + frag.append(" NULL AS CreatedBy,\n"); + frag.append(" NULL AS Modified,\n"); + frag.append(" NULL AS ModifiedBy,\n"); + frag.append(" NULL AS FilePath,\n"); + frag.append(" NULL AS SourceKey,\n"); + frag.append(" NULL AS SourceName\n"); + frag.append("WHERE 1 = 0"); + } + else + { + String union = ""; + frag.append("("); + for (FileListener fileListener : _fileListeners) + { + SQLFragment subselect = fileListener.listFilesQuery(); + if (subselect != null) + { + frag.append(union); + frag.append(subselect); + union = "UNION\n"; + } + } + frag.append(")"); + } + return frag; + } + + @Override + public void setFileRootSetViaStartupProperty(boolean fileRootSetViaStartupProperty) + { + _fileRootSetViaStartupProperty = fileRootSetViaStartupProperty; + } + + @Override + public boolean isFileRootSetViaStartupProperty() + { + return _fileRootSetViaStartupProperty; + } + + public ContainerListener getContainerListener() + { + return _containerListener; + } + + public Set> getNodes(boolean isShowOverridesOnly, @Nullable String browseUrl, Container c) + { + Set> children = new LinkedHashSet<>(); + + try { + java.nio.file.Path assayFilesRoot = getFileRootPath(c, ContentType.assayfiles); + if (NetworkDrive.exists(assayFilesRoot)) + { + Map node = createFileSetNode(c, ASSAY_FILES, assayFilesRoot); + node.put("default", false); + node.put("webdavURL", FilesWebPart.getRootPath(c, ASSAY_FILES).toString()); + children.add(node); + } + + AttachmentDirectory root = getMappedAttachmentDirectory(c, false); + if (root != null) + { + boolean isDefault = isUseDefaultRoot(c); + if (!isDefault || !isShowOverridesOnly) + { + ActionURL config = PageFlowUtil.urlProvider(AdminUrls.class).getProjectSettingsFileURL(c); + Map node = createFileSetNode(c, FILES_LINK, root.getFileSystemDirectoryPath()); + node.put("default", isUseDefaultRoot(c)); + node.put("configureURL", config.getEncodedLocalURIString()); + node.put("browseURL", browseUrl); + node.put("webdavURL", FilesWebPart.getRootPath(c, FILES_LINK).toString()); + + children.add(node); + } + } + + for (AttachmentDirectory fileSet : getRegisteredDirectories(c)) + { + ActionURL config = new ActionURL(FileContentController.ShowAdminAction.class, c); + Map node = createFileSetNode(c, fileSet.getName(), fileSet.getFileSystemDirectoryPath()); + node.put("configureURL", config.getEncodedLocalURIString()); + node.put("browseURL", browseUrl); + node.put("webdavURL", FilesWebPart.getRootPath(c, FILE_SETS_LINK, fileSet.getName()).toString()); + node.put("rootType", "fileset"); + + children.add(node); + } + + PipeRoot pipeRoot = PipelineService.get().findPipelineRoot(c); + if (pipeRoot != null) + { + boolean isDefault = PipelineService.get().hasSiteDefaultRoot(c); + if (!isDefault || !isShowOverridesOnly) + { + ActionURL config = PageFlowUtil.urlProvider(PipelineUrls.class).urlSetup(c); + ActionURL pipelineBrowse = PageFlowUtil.urlProvider(PipelineUrls.class).urlBrowse(c, null); + Map node = createFileSetNode(c, PIPELINE_LINK, pipeRoot.getRootNioPath()); + node.put("default", isDefault ); + node.put("configureURL", config.getEncodedLocalURIString()); + node.put("browseURL", pipelineBrowse.getEncodedLocalURIString()); + node.put("webdavURL", FilesWebPart.getRootPath(c, PIPELINE_LINK).toString()); + + children.add(node); + } + } + } + catch (IOException | UnsetRootDirectoryException ignored) {} + return children; + } + + protected Map createFileSetNode(Container container, String name, java.nio.file.Path dir) + { + Map node = new HashMap<>(); + if (dir != null) + { + node.put("name", name); + node.put("path", FileUtil.getAbsolutePath(container, dir)); + node.put("leaf", true); + } + return node; + } + + public String getAbsolutePathFromDataFileUrl(String dataFileUrl, Container container) + { + return FileUtil.getAbsolutePath(container, FileUtil.createUri(dataFileUrl)); + } + + @Nullable + @Override + public URI getWebDavUrl(@NotNull java.nio.file.Path path, @NotNull Container container, @NotNull PathType type) + { + PipeRoot root = PipelineService.get().getPipelineRootSetting(container); + java.nio.file.Path assayFilesPath = getFileRootPath(container, ContentType.assayfiles); + path = path.toAbsolutePath(); + String relPath = null; + URI rootWebDavUrl = null; + + try + { + // currently, only report if the file is under the parent container + if (root != null && root.isUnderRoot(path)) + { + relPath = root.relativePath(path); + rootWebDavUrl = root.getWebdavURL(); + } + else if (assayFilesPath != null && URIUtil.isDescendant(assayFilesPath.toUri(), path.toUri())) + { + relPath = assayFilesPath.relativize(path).toString(); + rootWebDavUrl = FilesWebPart.getRootPath(container, ASSAY_FILES); + } + + if (relPath != null) + { + relPath = Path.parse(FilenameUtils.separatorsToUnix(relPath)).encode(); + + return switch (type) + { + case folderRelative -> new URI(relPath); + case serverRelative -> new URI(rootWebDavUrl + (rootWebDavUrl.getPath().endsWith("/") ? "" : "/") + relPath); + case full -> new URI(AppProps.getInstance().getBaseServerUrl() + rootWebDavUrl + (rootWebDavUrl.getPath().endsWith("/") ? "" : "/") + relPath); + }; + } + } + catch (InvalidPathException | URISyntaxException e) + { + _log.error("Invalid WebDav URL from: " + path, e); + } + + return null; + } + + @Override + public String getDataFileRelativeFileRootPath(@NotNull String dataFileUrl, Container container) + { + Set> children = getNodes(false, null, container); + String filesRoot = null; // the path for @files + for (Map child : children) + { + String rootName = (String) child.get("name"); + String rootPath = (String) child.get("path"); + + // skip default @pipeline, which is the same as @files + if (PIPELINE_LINK.equals(rootName)) + { + if((boolean) child.get("default") || rootPath.equals(filesRoot)) + continue; + } + + if (FILES_LINK.equals(rootName)) + filesRoot = rootPath; + + String absoluteFilePath = getAbsolutePathFromDataFileUrl(dataFileUrl, container); + if (StringUtils.startsWith(absoluteFilePath, rootPath)) + { + String offset = absoluteFilePath.replace(rootPath, "").replace("\\", "/"); + int lastSlash = offset.lastIndexOf("/"); + if (lastSlash <= 0) + return "/"; + else + return offset.substring(0, lastSlash); + } + } + return null; + } + + @Override + public void ensureFileData(@NotNull ExpDataTable table) + { + Container container = table.getUserSchema().getContainer(); + // The current user may not have insert permission, and they didn't necessarily upload the files anyway + User user = User.getAdminServiceUser(); + QueryUpdateService qus = table.getUpdateService(); + if (qus == null) + { + throw new IllegalArgumentException("getUpdateServer() returned null from " + table); + } + + synchronized (_fileDataUpToDateCache) + { + if (_fileDataUpToDateCache.get(container) != null) // already synced in the past 5 minutes, skip + return; + + _fileDataUpToDateCache.put(container, true); + } + + List existingDataFileUrls = getDataFileUrls(container); + Collection filesets = getRegisteredDirectories(container); + Set> children = getNodes(false, null, container); + String filesRoot = null; // the path for @files + for (Map child : children) + { + String rootName = (String) child.get("name"); + String rootPathVal = (String) child.get("path"); + + // skip default @pipeline, which is the same as @files + if (PIPELINE_LINK.equals(rootName)) + { + if((boolean) child.get("default") || rootPathVal.equals(filesRoot)) + continue; + } + + if (FILES_LINK.equals(rootName)) + filesRoot = rootPathVal; + + String rootDavUrl = (String) child.get("webdavURL"); + + WebdavResource resource = getResource(rootDavUrl); + if (resource == null) + continue; + + List> rows = new ArrayList<>(); + BatchValidationException errors = new BatchValidationException(); + File file = resource.getFile(); + + if (file == null) + { + String rootType = (String) child.get("rootType"); + if ("fileset".equals(rootType)) + { + for (AttachmentDirectory fileset : filesets) + { + if (fileset.getName().equals(rootName)) + { + try + { + file = fileset.getFileSystemDirectory(); + } + catch (MissingRootDirectoryException e) + { + _log.error("Unable to list files for fileset: " + rootName, e); + } + break; + } + } + } + } + + if (file == null) + return; + + try (var ignore = SpringActionController.ignoreSqlUpdates()) + { + java.nio.file.Path rootPath = file.toPath(); + + try (Stream pathStream = Files.walk(rootPath, 100)) // prevent symlink loop + { + pathStream + .filter(path -> !Files.isSymbolicLink(path) && path.compareTo(rootPath) != 0) // exclude symlink & root + .forEach(path -> { + if (!containsUrlOrVariation(existingDataFileUrls, path)) + rows.add(new CaseInsensitiveHashMap<>(Collections.singletonMap("DataFileUrl", path.toUri().toString()))); + }); + } + + qus.insertRows(user, container, rows, errors, null, null); + } + catch (Exception e) + { + _log.error("Error listing content of directory: " + file.getAbsolutePath(), e); + } + } + } + + + @Override + public void addZiploaderPattern(DirectoryPattern directoryPattern) + { + _ziploaderPattern.add(directoryPattern); + } + + @Override + public List getZiploaderPatterns(Container container) + { + List registeredPatterns = new ArrayList<>(); + for(Module module : container.getActiveModules()) + { + _ziploaderPattern.forEach(p -> { + if(p.getModule().getName().equalsIgnoreCase(module.getName())) + registeredPatterns.add(p); + }); + } + return registeredPatterns; + } + + public List getDataFileUrls(Container container) + { + SimpleFilter filter = SimpleFilter.createContainerFilter(container); + filter.addCondition(FieldKey.fromParts("DataFileUrl"), null, CompareType.NONBLANK); + TableSelector selector = new TableSelector(ExperimentService.get().getTinfoData(), Collections.singleton("DataFileUrl"), filter, null); + return selector.getArrayList(String.class); + } + + public Path getPath(String uri) + { + Path path = Path.decode(uri); + + if (!path.startsWith(WebdavService.getPath()) && path.contains(WebdavService.getPath().getName())) + { + String newPath = path.toString(); + int idx = newPath.indexOf(WebdavService.getPath().toString()); + + if (idx != -1) + { + newPath = newPath.substring(idx); + path = Path.parse(newPath); + } + } + return path; + } + + @Nullable + public WebdavResource getResource(String uri) + { + Path path = getPath(uri); + return WebdavService.get().getResolver().lookup(path); + } + + public static void throwIfPathNotFile(java.nio.file.Path path, Container container) + { + if (null == path) + { + throw new RuntimeException("No path to evaluate in " + container.getPath()); + } + if (FileUtil.hasCloudScheme(path)) + { + throw new RuntimeException("Cannot get File object from Cloud File Root in " + container.getPath()); + } + } + + private boolean containsUrlOrVariation(List existingUrls, java.nio.file.Path path) + { + String url = path.toUri().toString(); + if (existingUrls.contains(url)) + return true; + + boolean urlHasTrailingSlash = (Files.isDirectory(path) && (url.endsWith("/") || url.endsWith(File.pathSeparator))); + if (urlHasTrailingSlash && existingUrls.contains(url.substring(0, url.length() - 1))) + return true; + + if (!FileUtil.hasCloudScheme(path)) + { + File file = path.toFile(); + String legacyUrl = file.toURI().toString(); + if (existingUrls.contains(legacyUrl)) // Legacy URI format (file:/users/...) + return true; + + return existingUrls.contains(file.getPath()); + } + return false; + } + + @Override + public File getMoveTargetFile(String absoluteFilePath, @NotNull Container sourceContainer, @NotNull Container targetContainer) + { + if (absoluteFilePath == null) + return null; + + File file = new File(absoluteFilePath); + if (!NetworkDrive.exists(file)) + { + _log.warn("File '" + absoluteFilePath + "' not found and cannot be moved"); + return null; + } + + File sourceFileRoot = getFileRoot(sourceContainer); + if (sourceFileRoot == null) + return null; + + String sourceRootPath = sourceFileRoot.getAbsolutePath(); + if (!absoluteFilePath.startsWith(sourceRootPath)) + { + _log.warn("File '" + absoluteFilePath + "' not currently located in source folder '" + sourceRootPath + "'. Not moving."); + return null; + } + File targetFileRoot = getFileRoot(targetContainer); + if (targetFileRoot == null) + return null; + + String targetPath = absoluteFilePath.replace(sourceRootPath, targetFileRoot.getAbsolutePath()); + File targetFile = new File(targetPath); + return FileUtil.findUniqueFileName(file.getName(), targetFile.getParentFile()); + } + + @Override + public void addDynamicWarnings(@NotNull Warnings warnings, @Nullable ViewContext context, boolean showAllWarnings) + { + if (_problematicFileRootMessage != null && context != null && ContainerManager.getRoot().hasPermission(context.getUser(), AdminOperationsPermission.class)) + { + warnings.add(DOM.createHtmlFragment(_problematicFileRootMessage, " ", DOM.A(at(href, PageFlowUtil.urlProvider(AdminUrls.class).getFilesSiteSettingsURL()), "Configure File System Access"))); + } + else if (showAllWarnings) + { + try + { + warnings.add(HtmlString.of("Configured site-wide file root " + getDefaultRoot() + " does not exist. Falling back to " + getDefaultRoot())); + } + catch (IOException ignored) {} + } + } + + // Cache with short-lived entries so that exp.files can perform reasonably + private static final Cache _fileDataUpToDateCache = CacheManager.getCache(CacheManager.UNLIMITED, 5 * CacheManager.MINUTE, "Files"); + + @TestWhen(TestWhen.When.BVT) + public static class TestCase extends AssertionError + { + private static final String TRICKY_CHARACTERS_FOR_PROJECT_NAMES = "\u2603~!@$&()_+{}-=[],.#\u00E4\u00F6\u00FC"; + + private static final String PROJECT1 = "FileRootTestProject1" + TRICKY_CHARACTERS_FOR_PROJECT_NAMES; + private static final String PROJECT1_SUBFOLDER1 = "Subfolder1"; + private static final String PROJECT1_SUBFOLDER2 = "Subfolder2" + TRICKY_CHARACTERS_FOR_PROJECT_NAMES; + private static final String PROJECT1_SUBSUBFOLDER = "SubSubfolder"; + private static final String PROJECT1_SUBSUBFOLDER_SIBLING = "SubSubfolderSibling"; + private static final String PROJECT2 = "FileRootTestProject2"; + + private static final String FILE_ROOT_SUFFIX = "_FileRootTest"; + private static final String TXT_FILE = "FileContentTestFile.txt"; + + private Map _expectedPaths; + + @Test + public void fileRootsTest() + { + //pre-clean + cleanup(); + + _expectedPaths = new HashMap<>(); + + FileContentService svc = FileContentService.get(); + Assert.assertNotNull(svc); + + Container project1 = ContainerManager.createContainer(ContainerManager.getRoot(), PROJECT1, TestContext.get().getUser()); + _expectedPaths.put(project1, null); + + Container project2 = ContainerManager.createContainer(ContainerManager.getRoot(), PROJECT2, TestContext.get().getUser()); + _expectedPaths.put(project2, null); + + Container subfolder1 = ContainerManager.createContainer(project1, PROJECT1_SUBFOLDER1, TestContext.get().getUser()); + _expectedPaths.put(subfolder1, null); + + Container subfolder2 = ContainerManager.createContainer(project1, PROJECT1_SUBFOLDER2, TestContext.get().getUser()); + _expectedPaths.put(subfolder2, null); + + Container subsubfolder = ContainerManager.createContainer(subfolder1, PROJECT1_SUBSUBFOLDER, TestContext.get().getUser()); + _expectedPaths.put(subsubfolder, null); + + //set custom root on project, then expect children to inherit + File testRoot = getTestRoot(); + + svc.setFileRoot(project1, testRoot); + _expectedPaths.put(project1, testRoot); + + //the subfolder should inherit from the parent + _expectedPaths.put(subfolder1, new File(testRoot, subfolder1.getName())); + assertPathsEqual("Incorrect values returned by getDefaultRoot", _expectedPaths.get(subfolder1), svc.getDefaultRoot(subfolder1, false)); + assertPathsEqual("Subfolder1 has incorrect root", _expectedPaths.get(subfolder1), svc.getFileRoot(subfolder1)); + + _expectedPaths.put(subfolder2, new File(testRoot, subfolder2.getName())); + assertPathsEqual("Incorrect values returned by getDefaultRoot", _expectedPaths.get(subfolder2), svc.getDefaultRoot(subfolder2, false)); + assertPathsEqual("Subfolder2 has incorrect root", _expectedPaths.get(subfolder2), svc.getFileRoot(subfolder2)); + + _expectedPaths.put(subsubfolder, new File(_expectedPaths.get(subfolder1), subsubfolder.getName())); + assertPathsEqual("Incorrect values returned by getDefaultRoot", _expectedPaths.get(subsubfolder), svc.getDefaultRoot(subsubfolder, false)); + assertPathsEqual("SubSubfolder has incorrect root", _expectedPaths.get(subsubfolder), svc.getFileRoot(subsubfolder)); + + //override root on 1st child, expect children of that folder to inherit + _expectedPaths.put(subfolder1, new File(testRoot, "CustomSubfolder")); + _expectedPaths.get(subfolder1).mkdirs(); + svc.setFileRoot(subfolder1, _expectedPaths.get(subfolder1)); + assertPathsEqual("SubSubfolder has incorrect root", new File(_expectedPaths.get(subfolder1), subsubfolder.getName()), svc.getFileRoot(subsubfolder)); + + //reset project, we assume overridden child roots to remain the same + svc.setFileRoot(project1, null); + assertPathsEqual("Subfolder1 has incorrect root", _expectedPaths.get(subfolder1), svc.getFileRoot(subfolder1)); + assertPathsEqual("SubSubfolder has incorrect root", new File(_expectedPaths.get(subfolder1), subsubfolder.getName()), svc.getFileRoot(subsubfolder)); + + } + + private void assertPathsEqual(String msg, File expected, File actual) + { + String expectedPath = FileUtil.getAbsoluteCaseSensitiveFile(expected).getPath(); + String actualPath = FileUtil.getAbsoluteCaseSensitiveFile(actual).getPath(); + Assert.assertEquals(msg, expectedPath, actualPath); + } + + private File getTestRoot() + { + FileContentService svc = FileContentService.get(); + File siteRoot = svc.getSiteDefaultRoot(); + File testRoot = new File(siteRoot, FILE_ROOT_SUFFIX); + testRoot.mkdirs(); + Assert.assertTrue("Unable to create test file root", NetworkDrive.exists(testRoot)); + + return testRoot; + } + + @Test + //when we move a folder, we expect child files to follow, and expect + // any file paths stored in the DB to also get updated + public void testFolderMove() throws Exception + { + //pre-clean + cleanup(); + + _expectedPaths = new HashMap<>(); + + FileContentService svc = FileContentService.get(); + Assert.assertNotNull(svc); + + Container project1 = ContainerManager.createContainer(ContainerManager.getRoot(), PROJECT1, TestContext.get().getUser()); + _expectedPaths.put(project1, null); + + Container project2 = ContainerManager.createContainer(ContainerManager.getRoot(), PROJECT2, TestContext.get().getUser()); + _expectedPaths.put(project2, null); + + Container subfolder1 = ContainerManager.createContainer(project1, PROJECT1_SUBFOLDER1, TestContext.get().getUser()); + _expectedPaths.put(subfolder1, null); + + Container subfolder2 = ContainerManager.createContainer(project1, PROJECT1_SUBFOLDER2, TestContext.get().getUser()); + _expectedPaths.put(subfolder2, null); + + Container subsubfolder = ContainerManager.createContainer(subfolder1, PROJECT1_SUBSUBFOLDER, TestContext.get().getUser()); + Container subsubfolderSibling = ContainerManager.createContainer(subfolder1, PROJECT1_SUBSUBFOLDER_SIBLING, TestContext.get().getUser()); + _expectedPaths.put(subsubfolder, null); + + //create a test file that we will follow + File fileRoot = svc.getFileRoot(subsubfolder, ContentType.files); + fileRoot.mkdirs(); + + File childFile = new File(fileRoot, TXT_FILE); + childFile.createNewFile(); + + ExpData data = ExperimentService.get().createData(subsubfolder, UPLOADED_FILE); + data.setDataFileURI(childFile.toPath().toUri()); + data.save(TestContext.get().getUser()); + + ExpProtocol protocol = ExperimentService.get().createExpProtocol(subsubfolder, ExpProtocol.ApplicationType.ProtocolApplication, "DummyProtocol"); + protocol = ExperimentService.get().insertSimpleProtocol(protocol, TestContext.get().getUser()); + + ExpRun expRun = ExperimentService.get().createExperimentRun(subsubfolder, "DummyRun"); + expRun.setProtocol(protocol); + expRun.setFilePathRootPath(childFile.getParentFile().toPath()); + + ViewBackgroundInfo info = new ViewBackgroundInfo(subsubfolder, TestContext.get().getUser(), null); + ExpRun run = ExperimentService.get().saveSimpleExperimentRun( + expRun, + Collections.emptyMap(), + Collections.singletonMap(data, "Data"), + Collections.emptyMap(), + Collections.emptyMap(), + Collections.emptyMap(), + info, + _log, + false); + + Assert.assertTrue("File not found: " + childFile.getPath(), NetworkDrive.exists(childFile)); + ContainerManager.move(subsubfolder, subfolder2, TestContext.get().getUser()); + Container movedSubfolder = ContainerManager.getChild(subfolder2, subsubfolder.getName()); + + _expectedPaths.put(movedSubfolder, new File(svc.getFileRoot(subfolder2), movedSubfolder.getName())); + assertPathsEqual("Incorrect values returned by getDefaultRoot", _expectedPaths.get(movedSubfolder), svc.getDefaultRoot(movedSubfolder, false)); + assertPathsEqual("SubSubfolder has incorrect root", _expectedPaths.get(movedSubfolder), svc.getFileRoot(movedSubfolder)); + + File expectedFile = new File(svc.getFileRoot(movedSubfolder, ContentType.files), TXT_FILE); + Assert.assertTrue("File was not moved, expected: " + expectedFile.getPath(), NetworkDrive.exists(expectedFile)); + + ExpData movedData = ExperimentService.get().getExpData(data.getRowId()); + Assert.assertNotNull(movedData); + + // Reload the run after it's path has hopefully been updated + expRun = ExperimentService.get().getExpRun(expRun.getRowId()); + + assertPathsEqual("Incorrect data file path", expectedFile, FileUtil.stringToPath(movedSubfolder, movedData.getDataFileUrl()).toFile()); + assertPathsEqual("Incorrect run root path", expectedFile.getParentFile(), expRun.getFilePathRoot()); + + // Issue 38206 - file paths get mangled with multiple folder moves + ContainerManager.move(subsubfolderSibling, subfolder2, TestContext.get().getUser()); + + // Reload the run after it's path has hopefully NOT been updated + expRun = ExperimentService.get().getExpRun(expRun.getRowId()); + assertPathsEqual("Incorrect run root path", expectedFile.getParentFile(), expRun.getFilePathRoot()); + } + + @Test + public void testWorkbooksAndTabs() + { + //pre-clean + cleanup(); + + FileContentService svc = FileContentService.get(); + Assert.assertNotNull(svc); + + Container project1 = ContainerManager.createContainer(ContainerManager.getRoot(), PROJECT1, TestContext.get().getUser()); + + Container workbook = ContainerManager.createContainer(project1, null, null, null, WorkbookContainerType.NAME, TestContext.get().getUser()); + File expectedWorkbookRoot = new File(svc.getFileRoot(project1), workbook.getName()); + assertPathsEqual("Workbook has incorrect file root", expectedWorkbookRoot, svc.getFileRoot(workbook)); + + Container tab = ContainerManager.createContainer(project1, "tab", null, null, TabContainerType.NAME, TestContext.get().getUser()); + File expectedTabRoot = new File(svc.getFileRoot(project1), tab.getName()); + assertPathsEqual("Folder tab has incorrect file root", expectedTabRoot, svc.getFileRoot(tab)); + } + + /** + * Test that the Site Settings can be configured from startup properties + */ + @Test + public void testStartupPropertiesForSiteRootSettings() throws IOException + { + // save the original Site Root File settings so that we can restore them when this test is done + File originalSiteRootFile = FileContentService.get().getSiteDefaultRoot(); + + // create the new site root file to test with as a child of the current site root file so that we know it is in a dir that exist + String originalSiteRootFilePath = originalSiteRootFile.getAbsolutePath(); + File testSiteRootFile = new File(originalSiteRootFilePath, "testSiteRootFile"); + testSiteRootFile.createNewFile(); + + ModuleLoader.getInstance().handleStartupProperties(new RandomSiteSettingsPropertyHandler(){ + @Override + public @NotNull Collection getStartupPropertyEntries() + { + return List.of(new StartupPropertyEntry("siteFileRoot", testSiteRootFile.getAbsolutePath(), "startup", SCOPE_SITE_SETTINGS)); + } + + @Override + public boolean performChecks() + { + return false; + } + }); + + // now check that the expected changes occurred to the Site Root File settings on the server + File newSiteRootFile = FileContentService.get().getSiteDefaultRoot(); + Assert.assertEquals("The expected change in Site Root File was not found", testSiteRootFile.getAbsolutePath(), newSiteRootFile.getAbsolutePath()); + + // restore the Site Root File server settings to how they were originally + FileContentService.get().setSiteDefaultRoot(originalSiteRootFile, null); + testSiteRootFile.delete(); + } + + @After + public void cleanup() + { + FileContentService svc = FileContentService.get(); + Assert.assertNotNull(svc); + + deleteContainerAndFiles(svc, ContainerManager.getForPath(PROJECT1)); + deleteContainerAndFiles(svc, ContainerManager.getForPath(PROJECT2)); + + File testRoot = getTestRoot(); + if (NetworkDrive.exists(testRoot)) + { + FileUtil.deleteDir(testRoot); + } + } + + private void deleteContainerAndFiles(FileContentService svc, @Nullable Container c) + { + if (c != null) + { + ContainerManager.deleteAll(c, TestContext.get().getUser()); + + File file1 = svc.getFileRoot(c); + if (NetworkDrive.exists(file1)) + { + FileUtil.deleteDir(file1); + } + } + } + } +} From a2ffc20eb181ddd5e9afdf5671713f05bb0aa784 Mon Sep 17 00:00:00 2001 From: XingY Date: Thu, 8 Jan 2026 17:10:43 -0800 Subject: [PATCH 02/14] crlf --- .../org/labkey/api/exp/query/ExpSchema.java | 1666 +++---- .../labkey/api/files/FileContentService.java | 724 +-- .../org/labkey/api/files/FileListener.java | 204 +- .../api/files/TableUpdaterFileListener.java | 922 ++-- .../experiment/FileLinkFileListener.java | 676 +-- .../filecontent/FileContentServiceImpl.java | 3948 ++++++++--------- 6 files changed, 4070 insertions(+), 4070 deletions(-) diff --git a/api/src/org/labkey/api/exp/query/ExpSchema.java b/api/src/org/labkey/api/exp/query/ExpSchema.java index 72ffff090c6..62b4ba7b661 100644 --- a/api/src/org/labkey/api/exp/query/ExpSchema.java +++ b/api/src/org/labkey/api/exp/query/ExpSchema.java @@ -1,833 +1,833 @@ -/* - * Copyright (c) 2009-2019 LabKey Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.labkey.api.exp.query; - -import org.jetbrains.annotations.NotNull; -import org.jetbrains.annotations.Nullable; -import org.labkey.api.collections.CaseInsensitiveHashSet; -import org.labkey.api.compliance.ComplianceService; -import org.labkey.api.data.ColumnInfo; -import org.labkey.api.data.Container; -import org.labkey.api.data.ContainerFilter; -import org.labkey.api.data.ContainerManager; -import org.labkey.api.data.EnumTableInfo; -import org.labkey.api.data.ForeignKey; -import org.labkey.api.data.TableInfo; -import org.labkey.api.data.UnionContainerFilter; -import org.labkey.api.exp.api.ExpRun; -import org.labkey.api.exp.api.ExpSampleType; -import org.labkey.api.exp.api.ExperimentService; -import org.labkey.api.exp.api.SampleTypeService; -import org.labkey.api.exp.property.DomainProperty; -import org.labkey.api.exp.property.Lookup; -import org.labkey.api.module.Module; -import org.labkey.api.ontology.KindOfQuantity; -import org.labkey.api.ontology.Unit; -import org.labkey.api.query.DefaultSchema; -import org.labkey.api.query.FieldKey; -import org.labkey.api.query.LookupForeignKey; -import org.labkey.api.query.QuerySchema; -import org.labkey.api.query.QuerySettings; -import org.labkey.api.query.QueryView; -import org.labkey.api.query.SchemaKey; -import org.labkey.api.security.User; -import org.labkey.api.security.permissions.InsertPermission; -import org.labkey.api.security.permissions.ReadPermission; -import org.labkey.api.util.StringExpression; -import org.labkey.api.view.ActionURL; -import org.labkey.api.view.ViewContext; -import org.springframework.validation.BindException; - -import java.util.ArrayList; -import java.util.Arrays; -import java.util.Collections; -import java.util.EnumSet; -import java.util.LinkedHashSet; -import java.util.List; -import java.util.Objects; -import java.util.Set; -import java.util.TreeSet; -import java.util.stream.Collectors; - -public class ExpSchema extends AbstractExpSchema -{ - public static final String EXPERIMENTS_MEMBERSHIP_FOR_RUN_TABLE_NAME = "ExperimentsMembershipForRun"; - public static final String DATA_CLASS_CATEGORY_TABLE = "DataClassCategoryType"; - public static final String SAMPLE_STATE_TYPE_TABLE = "SampleStateType"; - public static final String SAMPLE_TYPE_CATEGORY_TABLE = "SampleTypeCategoryType"; - public static final String MEASUREMENT_UNITS_TABLE = "MeasurementUnits"; - public static final String SAMPLE_FILES_TABLE = "StaleSampleFiles"; - - public static final SchemaKey SCHEMA_EXP = SchemaKey.fromParts(ExpSchema.SCHEMA_NAME); - public static final SchemaKey SCHEMA_EXP_DATA = SchemaKey.fromString(SCHEMA_EXP, ExpSchema.NestedSchemas.data.name()); - public static final SchemaKey SCHEMA_EXP_MATERIALS = SchemaKey.fromString(SCHEMA_EXP, ExpSchema.NestedSchemas.materials.name()); - - private static final Set ADDITIONAL_SOURCES_AUDIT_FIELDS = new CaseInsensitiveHashSet("Name", "RowId"); - - public enum NestedSchemas - { - data, - materials - } - - public enum TableType - { - Runs - { - @Override - public TableInfo createTable(ExpSchema expSchema, String queryName, ContainerFilter cf) - { - ExpRunTable ret = ExperimentService.get().createRunTable(Runs.toString(), expSchema, cf); - return expSchema.setupTable(ret); - } - }, - Data - { - @Override - public TableInfo createTable(ExpSchema expSchema, String queryName, ContainerFilter cf) - { - ExpDataTable ret = ExperimentService.get().createDataTable(Data.toString(), expSchema, cf); - return expSchema.setupTable(ret); - } - }, - DataInputs - { - @Override - public TableInfo createTable(ExpSchema expSchema, String queryName, ContainerFilter cf) - { - ExpDataInputTable ret = ExperimentService.get().createDataInputTable(DataInputs.toString(), expSchema, cf); - return expSchema.setupTable(ret); - } - }, - DataProtocolInputs - { - @Override - public TableInfo createTable(ExpSchema expSchema, String queryName, ContainerFilter cf) - { - ExpDataProtocolInputTable ret = ExperimentService.get().createDataProtocolInputTable(DataProtocolInputs.toString(), expSchema, cf); - return expSchema.setupTable(ret); - } - }, - Materials - { - @Override - public TableInfo createTable(ExpSchema expSchema, String queryName, ContainerFilter cf) - { - ExpMaterialTable ret = ExperimentService.get().createMaterialTable(expSchema, cf, null); - return expSchema.setupTable(ret); - } - }, - MaterialInputs - { - @Override - public TableInfo createTable(ExpSchema expSchema, String queryName, ContainerFilter cf) - { - ExpMaterialInputTable ret = ExperimentService.get().createMaterialInputTable(MaterialInputs.toString(), expSchema, cf); - return expSchema.setupTable(ret); - } - }, - MaterialProtocolInputs - { - @Override - public TableInfo createTable(ExpSchema expSchema, String queryName, ContainerFilter cf) - { - ExpMaterialProtocolInputTable ret = ExperimentService.get().createMaterialProtocolInputTable(MaterialProtocolInputs.toString(), expSchema, cf); - return expSchema.setupTable(ret); - } - }, - Protocols - { - @Override - public TableInfo createTable(ExpSchema expSchema, String queryName, ContainerFilter cf) - { - ExpProtocolTable ret = ExperimentService.get().createProtocolTable(Protocols.toString(), expSchema, cf); - return expSchema.setupTable(ret); - } - }, - SampleSets - { - @Override - public TableInfo createTable(ExpSchema expSchema, String queryName, ContainerFilter cf) - { - ExpSampleTypeTable ret = ExperimentService.get().createSampleTypeTable(SampleSets.toString(), expSchema, cf); - return expSchema.setupTable(ret); - } - }, - DataClasses - { - @Override - public TableInfo createTable(ExpSchema expSchema, String queryName, ContainerFilter cf) - { - ExpDataClassTable ret = ExperimentService.get().createDataClassTable(DataClasses.toString(), expSchema, cf); - return expSchema.setupTable(ret); - } - }, - RunGroups - { - @Override - public TableInfo createTable(ExpSchema expSchema, String queryName, ContainerFilter cf) - { - ExpExperimentTable ret = ExperimentService.get().createExperimentTable(RunGroups.toString(), expSchema, cf); - return expSchema.setupTable(ret); - } - }, - RunGroupMap - { - @Override - public TableInfo createTable(ExpSchema expSchema, String queryName, ContainerFilter cf) - { - ExpRunGroupMapTable ret = ExperimentService.get().createRunGroupMapTable(RunGroupMap.toString(), expSchema, cf); - return expSchema.setupTable(ret); - } - }, - ProtocolApplications - { - @Override - public TableInfo createTable(ExpSchema expSchema, String queryName, ContainerFilter cf) - { - ExpProtocolApplicationTable result = ExperimentService.get().createProtocolApplicationTable(ProtocolApplications.toString(), expSchema, cf); - return expSchema.setupTable(result); - } - }, - QCFlags - { - @Override - public TableInfo createTable(ExpSchema expSchema, String queryName, ContainerFilter cf) - { - ExpQCFlagTable result = ExperimentService.get().createQCFlagsTable(QCFlags.toString(), expSchema, cf); - return expSchema.setupTable(result); - } - }, - Files - { - @Override - public TableInfo createTable(ExpSchema expSchema, String queryName, ContainerFilter cf) - { - ExpDataTable result = ExperimentService.get().createFilesTable(Files.toString(), expSchema); - return expSchema.setupTable(result); - } - }, - StaleSampleFiles - { - @Override - public TableInfo createTable(ExpSchema expSchema, String queryName, ContainerFilter cf) - { - return new ExpStaleSampleFilesTable(expSchema, cf); - } - }, - SampleStatus - { - @Override - public TableInfo createTable(ExpSchema expSchema, String queryName, ContainerFilter cf) - { - return ExperimentService.get().createSampleStatusTable(expSchema, cf); - } - }, - Fields - { - @Override - public TableInfo createTable(ExpSchema expSchema, String queryName, ContainerFilter cf) - { - return ExperimentService.get().createFieldsTable(expSchema, cf); - } - }, - PhiFields - { - @Override - public TableInfo createTable(ExpSchema expSchema, String queryName, ContainerFilter cf) - { - return ExperimentService.get().createPhiFieldsTable(expSchema, cf); - } - - @Override - public boolean includeTable() - { - return ComplianceService.get().isComplianceSupported(); - } - }; - public abstract TableInfo createTable(ExpSchema expSchema, String queryName, ContainerFilter cf); - - public boolean includeTable() - { - return true; - } - } - - public ExpTable getTable(TableType tableType) - { - return (ExpTable)getTable(tableType.toString()); - } - - public ExpTable getTable(TableType tableType, ContainerFilter cf) - { - return (ExpTable)getTable(tableType.toString(), cf); - } - - public ExpExperimentTable createExperimentsTableWithRunMemberships(ExpRun run, ContainerFilter cf) - { - ExpExperimentTable ret = ExperimentService.get().createExperimentTable(EXPERIMENTS_MEMBERSHIP_FOR_RUN_TABLE_NAME, this, cf); - setupTable(ret); - // Don't include exp.experiment rows that are assay batches - ret.setBatchProtocol(null); - ret.getMutableColumn(ExpExperimentTable.Column.RunCount).setHidden(true); - - ret.addExperimentMembershipColumn(run); - List defaultCols = new ArrayList<>(ret.getDefaultVisibleColumns()); - defaultCols.add(0, FieldKey.fromParts("RunMembership")); - defaultCols.remove(FieldKey.fromParts(ExpExperimentTable.Column.RunCount.name())); - ret.setDefaultVisibleColumns(defaultCols); - - return ret; - } - - // Use a holder pattern to initialize table names lazily, since ExpSchema gets referenced extremely early, before - // TableType.includeTable() might be ready to be called. e.g., ComplianceService isn't initialized yet. - private static final class TableNamesHolder - { - private static final Set tableNames; - - static - { - Set names = Arrays.stream(TableType.values()) - .filter(TableType::includeTable) - .map(TableType::toString) - .collect(Collectors.toCollection(LinkedHashSet::new)); - - names.add(DATA_CLASS_CATEGORY_TABLE); - names.add(SAMPLE_STATE_TYPE_TABLE); - names.add(SAMPLE_TYPE_CATEGORY_TABLE); - names.add(MEASUREMENT_UNITS_TABLE); - - tableNames = Collections.unmodifiableSet(names); - } - } - - public static final String SCHEMA_NAME = "exp"; - public static final String SCHEMA_DESCR = "Contains data about experiment runs, data files, materials, sample types, etc."; - - static public void register(final Module module) - { - DefaultSchema.registerProvider(SCHEMA_NAME, new DefaultSchema.SchemaProvider(module) - { - @Override - public boolean isAvailable(DefaultSchema schema, Module module) - { - return true; - } - - @Override - public QuerySchema createSchema(DefaultSchema schema, Module module) - { - return new ExpSchema(schema.getUser(), schema.getContainer()); - } - }); - } - - public SamplesSchema getSamplesSchema() - { - SamplesSchema schema = new SamplesSchema(this); - schema.setContainerFilter(_containerFilter); - return schema; - } - - public ExpSchema(User user, Container container) - { - super(SCHEMA_NAME, SCHEMA_DESCR, user, container, ExperimentService.get().getSchema()); - } - - @Override - public Set getTableNames() - { - return TableNamesHolder.tableNames; - } - - @Override - public TableInfo createTable(String name, ContainerFilter cf) - { - for (TableType tableType : TableType.values()) - { - if (tableType.name().equalsIgnoreCase(name) && tableType.includeTable()) - { - return tableType.createTable(this, tableType.name(), cf); - } - } - - if ("Experiments".equalsIgnoreCase(name)) - { - // Support "Experiments" as a legacy name for the RunGroups table - return TableType.RunGroups.createTable(this, name, cf); - } - if ("Datas".equalsIgnoreCase(name)) - { - /// Support "Datas" as a legacy name for the Data table - return TableType.Data.createTable(this, name, cf); - } - if (EXPERIMENTS_MEMBERSHIP_FOR_RUN_TABLE_NAME.equalsIgnoreCase(name)) - { - return createExperimentsTableWithRunMemberships(null, cf); - } - - if (DATA_CLASS_CATEGORY_TABLE.equalsIgnoreCase(name)) - { - return new EnumTableInfo<>(DataClassCategoryType.class, this, DataClassCategoryType::name, true, "Contains the list of available data class category types."); - } - - if (SAMPLE_STATE_TYPE_TABLE.equalsIgnoreCase(name)) - { - return new EnumTableInfo<>(SampleStateType.class, this, SampleStateType::name, true, "Contains the available sample state (status) types."); - } - - if (SAMPLE_TYPE_CATEGORY_TABLE.equalsIgnoreCase(name)) - { - return new EnumTableInfo<>(SampleTypeCategoryType.class, this, SampleTypeCategoryType::name, true, "Contains the list of available sample type category types."); - } - - if (MEASUREMENT_UNITS_TABLE.equalsIgnoreCase(name)) - { - // Create an EnumSet of the KindOfQuantity getCommonUnits - List commonUnits = KindOfQuantity.getSupportedUnits(); - EnumTableInfo table = new EnumTableInfo<>(Unit.class, EnumSet.copyOf(commonUnits), this, Unit::name, Unit::ordinal, false, "Contains the list of available units for measurements such as sample stored amounts."); - table.setPublicSchemaName(this.getName()); - table.setName(MEASUREMENT_UNITS_TABLE); - return table; - } - - return null; - } - - /** - * Exposed as EnumTableInfo. Update stateTypeEnum in qcStates.xsd if the enum values change. - */ - public enum SampleStateType - { - Available(Set.of(SampleTypeService.SampleOperations.values())), - Consumed(Set.of( - SampleTypeService.SampleOperations.EditMetadata, - SampleTypeService.SampleOperations.EditLineage, - SampleTypeService.SampleOperations.RemoveFromStorage, - SampleTypeService.SampleOperations.AddToPicklist, - SampleTypeService.SampleOperations.Delete, - SampleTypeService.SampleOperations.AddToWorkflow, - SampleTypeService.SampleOperations.RemoveFromWorkflow, - SampleTypeService.SampleOperations.AddAssayData, - SampleTypeService.SampleOperations.LinkToStudy, - SampleTypeService.SampleOperations.RecallFromStudy, - SampleTypeService.SampleOperations.Move - )), - Locked(Set.of( - SampleTypeService.SampleOperations.AddToPicklist - )); - - Set _permittedOps; - - SampleStateType(Set permittedOps) - { - _permittedOps = permittedOps; - } - - public Set getPermittedOps() - { - return _permittedOps; - } - } - - - /** - * Exposed as EnumTableInfo - * - */ - public enum DataClassCategoryType - { - registry(null), - media(null), - sources(ADDITIONAL_SOURCES_AUDIT_FIELDS); - - public final Set additionalAuditFields; - - DataClassCategoryType(@Nullable Set addlAuditFields) - { - this.additionalAuditFields = addlAuditFields; - } - - public static DataClassCategoryType fromString(String typeVal) { - for (DataClassCategoryType type : DataClassCategoryType.values()) { - if (type.name().equalsIgnoreCase(typeVal)) { - return type; - } - } - return null; - } - } - - public enum SampleTypeCategoryType - { - media; - - public static SampleTypeCategoryType fromString(String typeVal) { - for (SampleTypeCategoryType type : SampleTypeCategoryType.values()) { - if (type.name().equalsIgnoreCase(typeVal)) { - return type; - } - } - return null; - } - } - - @Override - public Set getSchemaNames() - { - if (_restricted) - return Collections.emptySet(); - - Set names = new TreeSet<>(String.CASE_INSENSITIVE_ORDER); - names.addAll(super.getSchemaNames()); - names.add(NestedSchemas.materials.name()); - names.add(NestedSchemas.data.name()); - return names; - } - - public QuerySchema getSchema(NestedSchemas schema) - { - return getSchema(schema.name()); - } - - @Override - public QuerySchema getSchema(String name) - { - if (_restricted) - return null; - - // CONSIDER: also support hidden "samples" schema ? - if (name.equals(NestedSchemas.materials.name())) - return new SamplesSchema(SchemaKey.fromParts(getName(), NestedSchemas.materials.name()), getUser(), getContainer()); - - if (name.equals(NestedSchemas.data.name())) - return new DataClassUserSchema(getContainer(), getUser()); - - return super.getSchema(name); - } - - public ExpDataTable getDatasTable() - { - return (ExpDataTable)getTable(TableType.Data); - } - - public ExpDataTable getDatasTable(boolean forWrite) - { - return (ExpDataTable)getTable(TableType.Data.toString(), null, true, forWrite); - } - - public ExpRunTable getRunsTable() - { - return (ExpRunTable)getTable(TableType.Runs.toString(), null, true, false); - } - - public ExpRunTable getRunsTable(boolean forWrite) - { - return (ExpRunTable)getTable(TableType.Runs.toString(), null, true, forWrite); - } - - - public ForeignKey getProtocolApplicationForeignKey(ContainerFilter cf) - { - return new ExperimentLookupForeignKey(null, null, ExpSchema.SCHEMA_NAME, TableType.ProtocolApplications.name(), "RowId", null) - { - @Override - public TableInfo getLookupTableInfo() - { - return getTable(TableType.ProtocolApplications, cf); - } - }; - } - - public ForeignKey getProtocolForeignKey(ContainerFilter cf, String targetColumnName) - { - return new LookupForeignKey(targetColumnName) - { - @Override - public TableInfo getLookupTableInfo() - { - return getTable(TableType.Protocols.toString(), ContainerFilter.getUnsafeEverythingFilter()); - } - }; - } - - public ForeignKey getMaterialForeignKey(ContainerFilter cf, String targetColumnName) - { - return new LookupForeignKey(targetColumnName) - { - @Override - public TableInfo getLookupTableInfo() - { - return getTable(TableType.Materials.toString(), cf); - } - - @Override - public StringExpression getURL(ColumnInfo parent) - { - return getURL(parent, true); - } - }; - } - - public ForeignKey getMaterialProtocolInputForeignKey(ContainerFilter cf) - { - return new ExperimentLookupForeignKey(null, null, ExpSchema.SCHEMA_NAME, TableType.MaterialProtocolInputs.name(), "RowId", null) - { - @Override - public TableInfo getLookupTableInfo() - { - return getTable(TableType.MaterialProtocolInputs, cf); - } - }; - } - - public ForeignKey getDataProtocolInputForeignKey(ContainerFilter cf) - { - return new ExperimentLookupForeignKey(null, null, ExpSchema.SCHEMA_NAME, TableType.DataProtocolInputs.name(), "RowId", null) - { - @Override - public TableInfo getLookupTableInfo() - { - return getTable(TableType.DataProtocolInputs, cf); - } - }; - } - - public ForeignKey getJobForeignKey() - { - return new LookupForeignKey("RowId", "RowId") - { - @Override - public TableInfo getLookupTableInfo() - { - QuerySchema pipeline = getDefaultSchema().getSchema("pipeline"); - if (null == pipeline) - return null; - return pipeline.getTable("Job", getDefaultContainerFilter()); - } - - @Override - public StringExpression getURL(ColumnInfo parent) - { - return getURL(parent, true); - } - }; - } - - @Deprecated - public ForeignKey getRunIdForeignKey() - { - return getRunIdForeignKey(null); - } - - public ForeignKey getRunIdForeignKey(ContainerFilter cf) - { - return new ExperimentLookupForeignKey(null, null, ExpSchema.SCHEMA_NAME, TableType.Runs.name(), "RowId", null) - { - @Override - public TableInfo getLookupTableInfo() - { - return getTable(TableType.Runs, cf); - } - }; - } - - @Deprecated - public ForeignKey getRunGroupIdForeignKey(final boolean includeBatches) - { - return getRunGroupIdForeignKey(null, includeBatches); - } - - /** @param includeBatches if false, then filter out run groups of type batch when doing the join */ - public ForeignKey getRunGroupIdForeignKey(ContainerFilter cf, final boolean includeBatches) - { - return new ExperimentLookupForeignKey(null, null, ExpSchema.SCHEMA_NAME, TableType.RunGroups.name(), "RowId", null) - { - @Override - public TableInfo getLookupTableInfo() - { - ContainerFilter cf = getLookupContainerFilter(); - String key = getClass().getName() + "/RunGroupIdForeignKey/" + includeBatches + "/" + cf.getCacheKey(); - // since getTable(forWrite=true) does not cache, cache this tableinfo using getCachedLookupTableInfo() - return ExpSchema.this.getCachedLookupTableInfo(key, this::createLookupTableInfo); - } - - @Override - protected ContainerFilter getLookupContainerFilter() - { - return Objects.requireNonNullElse(cf, ContainerFilter.Type.CurrentPlusProjectAndShared.create(ExpSchema.this)); - } - - private TableInfo createLookupTableInfo() - { - // CONSIDER: I wonder if this shouldn't be using UnionContainerFilter(cf, CurrentPlusProjectAndShared) - ExpExperimentTable result = (ExpExperimentTable) getTable(TableType.RunGroups.name(), getLookupContainerFilter(), true, true); - if (!includeBatches) - { - result.setBatchProtocol(null); - } - result.setLocked(true); - return result; - } - }; - } - - public ForeignKey getDataIdForeignKey(ContainerFilter cf) - { - return new ExperimentLookupForeignKey(null, null, ExpSchema.SCHEMA_NAME, TableType.Data.name(), "RowId", null) - { - @Override - public TableInfo getLookupTableInfo() - { - return getTable(TableType.Data, cf); - } - }; - } - - /** - * @param domainProperty the property on which the lookup is configured - */ - @NotNull - public ForeignKey getMaterialIdForeignKey(@Nullable ExpSampleType targetSampleType, @Nullable DomainProperty domainProperty, @Nullable ContainerFilter cfParent) - { - if (targetSampleType == null) - { - return new ExperimentLookupForeignKey(null, null, ExpSchema.SCHEMA_NAME, TableType.Materials.name(), "RowId", null) - { - @Override - public TableInfo getLookupTableInfo() - { - ContainerFilter cf = new ContainerFilter.SimpleContainerFilter(getSearchContainers(getContainer(), targetSampleType, domainProperty, getUser())); - if (null != cfParent) - cf = new UnionContainerFilter(cf, cfParent); - ExpTable result = getTable(TableType.Materials, cf); - return result; - } - }; - } - return getSamplesSchema().materialIdForeignKey(targetSampleType, domainProperty, cfParent); - } - - @NotNull - public static Set getSearchContainers(Container currentContainer, @Nullable ExpSampleType st, @Nullable DomainProperty dp, User user) - { - Set searchContainers = new LinkedHashSet<>(); - if (dp != null) - { - Lookup lookup = dp.getLookup(); - if (lookup != null && lookup.getContainer() != null) - { - Container lookupContainer = lookup.getContainer(); - if (lookupContainer.hasPermission(user, ReadPermission.class)) - { - // The property is specifically targeting a container, so look there and only there - searchContainers.add(lookup.getContainer()); - } - } - } - - if (searchContainers.isEmpty()) - { - // Default to looking in the current container - searchContainers.add(currentContainer); - if (st == null || (st.getContainer().isProject() && !currentContainer.isProject())) - { - Container c = currentContainer.getParent(); - // Recurse up the chain to the project - while (c != null && !c.isRoot()) - { - if (c.hasPermission(user, ReadPermission.class)) - { - searchContainers.add(c); - } - c = c.getParent(); - } - } - Container sharedContainer = ContainerManager.getSharedContainer(); - if (st == null || st.getContainer().equals(sharedContainer)) - { - if (sharedContainer.hasPermission(user, ReadPermission.class)) - { - searchContainers.add(ContainerManager.getSharedContainer()); - } - } - } - return searchContainers; - } - - public abstract static class ExperimentLookupForeignKey extends LookupForeignKey - { - public ExperimentLookupForeignKey(String pkColumnName) - { - super(pkColumnName); - } - - public ExperimentLookupForeignKey(ActionURL baseURL, String paramName, String schemaName, String tableName, String pkColumnName, String titleColumn) - { - super(baseURL, paramName, schemaName, tableName, pkColumnName, titleColumn); - } - - @Override - public StringExpression getURL(ColumnInfo parent) - { - return getURL(parent, true); - } - } - - @Override - public @NotNull QueryView createView(ViewContext context, @NotNull QuerySettings settings, BindException errors) - { - if (TableType.DataClasses.name().equalsIgnoreCase(settings.getQueryName())) - { - return new QueryView(this, settings, errors) - { - @Override - protected boolean canInsert() - { - TableInfo table = getTable(); - return table != null && table.hasPermission(getUser(), InsertPermission.class); - } - - @Override - public boolean showImportDataButton() - { - return false; - } - }; - } - - QueryView queryView = super.createView(context, settings, errors); - - if (TableType.Materials.name().equalsIgnoreCase(settings.getQueryName()) || - TableType.Data.name().equalsIgnoreCase(settings.getQueryName()) || - TableType.Protocols.name().equalsIgnoreCase(settings.getQueryName())) - { - // Use default delete button, but without showing the confirmation text - queryView.setShowDeleteButtonConfirmationText(false); - } - - return queryView; - } - - public enum DerivationDataScopeType - { - ChildOnly, - ParentOnly, - All - } -} +/* + * Copyright (c) 2009-2019 LabKey Corporation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.labkey.api.exp.query; + +import org.jetbrains.annotations.NotNull; +import org.jetbrains.annotations.Nullable; +import org.labkey.api.collections.CaseInsensitiveHashSet; +import org.labkey.api.compliance.ComplianceService; +import org.labkey.api.data.ColumnInfo; +import org.labkey.api.data.Container; +import org.labkey.api.data.ContainerFilter; +import org.labkey.api.data.ContainerManager; +import org.labkey.api.data.EnumTableInfo; +import org.labkey.api.data.ForeignKey; +import org.labkey.api.data.TableInfo; +import org.labkey.api.data.UnionContainerFilter; +import org.labkey.api.exp.api.ExpRun; +import org.labkey.api.exp.api.ExpSampleType; +import org.labkey.api.exp.api.ExperimentService; +import org.labkey.api.exp.api.SampleTypeService; +import org.labkey.api.exp.property.DomainProperty; +import org.labkey.api.exp.property.Lookup; +import org.labkey.api.module.Module; +import org.labkey.api.ontology.KindOfQuantity; +import org.labkey.api.ontology.Unit; +import org.labkey.api.query.DefaultSchema; +import org.labkey.api.query.FieldKey; +import org.labkey.api.query.LookupForeignKey; +import org.labkey.api.query.QuerySchema; +import org.labkey.api.query.QuerySettings; +import org.labkey.api.query.QueryView; +import org.labkey.api.query.SchemaKey; +import org.labkey.api.security.User; +import org.labkey.api.security.permissions.InsertPermission; +import org.labkey.api.security.permissions.ReadPermission; +import org.labkey.api.util.StringExpression; +import org.labkey.api.view.ActionURL; +import org.labkey.api.view.ViewContext; +import org.springframework.validation.BindException; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.EnumSet; +import java.util.LinkedHashSet; +import java.util.List; +import java.util.Objects; +import java.util.Set; +import java.util.TreeSet; +import java.util.stream.Collectors; + +public class ExpSchema extends AbstractExpSchema +{ + public static final String EXPERIMENTS_MEMBERSHIP_FOR_RUN_TABLE_NAME = "ExperimentsMembershipForRun"; + public static final String DATA_CLASS_CATEGORY_TABLE = "DataClassCategoryType"; + public static final String SAMPLE_STATE_TYPE_TABLE = "SampleStateType"; + public static final String SAMPLE_TYPE_CATEGORY_TABLE = "SampleTypeCategoryType"; + public static final String MEASUREMENT_UNITS_TABLE = "MeasurementUnits"; + public static final String SAMPLE_FILES_TABLE = "StaleSampleFiles"; + + public static final SchemaKey SCHEMA_EXP = SchemaKey.fromParts(ExpSchema.SCHEMA_NAME); + public static final SchemaKey SCHEMA_EXP_DATA = SchemaKey.fromString(SCHEMA_EXP, ExpSchema.NestedSchemas.data.name()); + public static final SchemaKey SCHEMA_EXP_MATERIALS = SchemaKey.fromString(SCHEMA_EXP, ExpSchema.NestedSchemas.materials.name()); + + private static final Set ADDITIONAL_SOURCES_AUDIT_FIELDS = new CaseInsensitiveHashSet("Name", "RowId"); + + public enum NestedSchemas + { + data, + materials + } + + public enum TableType + { + Runs + { + @Override + public TableInfo createTable(ExpSchema expSchema, String queryName, ContainerFilter cf) + { + ExpRunTable ret = ExperimentService.get().createRunTable(Runs.toString(), expSchema, cf); + return expSchema.setupTable(ret); + } + }, + Data + { + @Override + public TableInfo createTable(ExpSchema expSchema, String queryName, ContainerFilter cf) + { + ExpDataTable ret = ExperimentService.get().createDataTable(Data.toString(), expSchema, cf); + return expSchema.setupTable(ret); + } + }, + DataInputs + { + @Override + public TableInfo createTable(ExpSchema expSchema, String queryName, ContainerFilter cf) + { + ExpDataInputTable ret = ExperimentService.get().createDataInputTable(DataInputs.toString(), expSchema, cf); + return expSchema.setupTable(ret); + } + }, + DataProtocolInputs + { + @Override + public TableInfo createTable(ExpSchema expSchema, String queryName, ContainerFilter cf) + { + ExpDataProtocolInputTable ret = ExperimentService.get().createDataProtocolInputTable(DataProtocolInputs.toString(), expSchema, cf); + return expSchema.setupTable(ret); + } + }, + Materials + { + @Override + public TableInfo createTable(ExpSchema expSchema, String queryName, ContainerFilter cf) + { + ExpMaterialTable ret = ExperimentService.get().createMaterialTable(expSchema, cf, null); + return expSchema.setupTable(ret); + } + }, + MaterialInputs + { + @Override + public TableInfo createTable(ExpSchema expSchema, String queryName, ContainerFilter cf) + { + ExpMaterialInputTable ret = ExperimentService.get().createMaterialInputTable(MaterialInputs.toString(), expSchema, cf); + return expSchema.setupTable(ret); + } + }, + MaterialProtocolInputs + { + @Override + public TableInfo createTable(ExpSchema expSchema, String queryName, ContainerFilter cf) + { + ExpMaterialProtocolInputTable ret = ExperimentService.get().createMaterialProtocolInputTable(MaterialProtocolInputs.toString(), expSchema, cf); + return expSchema.setupTable(ret); + } + }, + Protocols + { + @Override + public TableInfo createTable(ExpSchema expSchema, String queryName, ContainerFilter cf) + { + ExpProtocolTable ret = ExperimentService.get().createProtocolTable(Protocols.toString(), expSchema, cf); + return expSchema.setupTable(ret); + } + }, + SampleSets + { + @Override + public TableInfo createTable(ExpSchema expSchema, String queryName, ContainerFilter cf) + { + ExpSampleTypeTable ret = ExperimentService.get().createSampleTypeTable(SampleSets.toString(), expSchema, cf); + return expSchema.setupTable(ret); + } + }, + DataClasses + { + @Override + public TableInfo createTable(ExpSchema expSchema, String queryName, ContainerFilter cf) + { + ExpDataClassTable ret = ExperimentService.get().createDataClassTable(DataClasses.toString(), expSchema, cf); + return expSchema.setupTable(ret); + } + }, + RunGroups + { + @Override + public TableInfo createTable(ExpSchema expSchema, String queryName, ContainerFilter cf) + { + ExpExperimentTable ret = ExperimentService.get().createExperimentTable(RunGroups.toString(), expSchema, cf); + return expSchema.setupTable(ret); + } + }, + RunGroupMap + { + @Override + public TableInfo createTable(ExpSchema expSchema, String queryName, ContainerFilter cf) + { + ExpRunGroupMapTable ret = ExperimentService.get().createRunGroupMapTable(RunGroupMap.toString(), expSchema, cf); + return expSchema.setupTable(ret); + } + }, + ProtocolApplications + { + @Override + public TableInfo createTable(ExpSchema expSchema, String queryName, ContainerFilter cf) + { + ExpProtocolApplicationTable result = ExperimentService.get().createProtocolApplicationTable(ProtocolApplications.toString(), expSchema, cf); + return expSchema.setupTable(result); + } + }, + QCFlags + { + @Override + public TableInfo createTable(ExpSchema expSchema, String queryName, ContainerFilter cf) + { + ExpQCFlagTable result = ExperimentService.get().createQCFlagsTable(QCFlags.toString(), expSchema, cf); + return expSchema.setupTable(result); + } + }, + Files + { + @Override + public TableInfo createTable(ExpSchema expSchema, String queryName, ContainerFilter cf) + { + ExpDataTable result = ExperimentService.get().createFilesTable(Files.toString(), expSchema); + return expSchema.setupTable(result); + } + }, + StaleSampleFiles + { + @Override + public TableInfo createTable(ExpSchema expSchema, String queryName, ContainerFilter cf) + { + return new ExpStaleSampleFilesTable(expSchema, cf); + } + }, + SampleStatus + { + @Override + public TableInfo createTable(ExpSchema expSchema, String queryName, ContainerFilter cf) + { + return ExperimentService.get().createSampleStatusTable(expSchema, cf); + } + }, + Fields + { + @Override + public TableInfo createTable(ExpSchema expSchema, String queryName, ContainerFilter cf) + { + return ExperimentService.get().createFieldsTable(expSchema, cf); + } + }, + PhiFields + { + @Override + public TableInfo createTable(ExpSchema expSchema, String queryName, ContainerFilter cf) + { + return ExperimentService.get().createPhiFieldsTable(expSchema, cf); + } + + @Override + public boolean includeTable() + { + return ComplianceService.get().isComplianceSupported(); + } + }; + public abstract TableInfo createTable(ExpSchema expSchema, String queryName, ContainerFilter cf); + + public boolean includeTable() + { + return true; + } + } + + public ExpTable getTable(TableType tableType) + { + return (ExpTable)getTable(tableType.toString()); + } + + public ExpTable getTable(TableType tableType, ContainerFilter cf) + { + return (ExpTable)getTable(tableType.toString(), cf); + } + + public ExpExperimentTable createExperimentsTableWithRunMemberships(ExpRun run, ContainerFilter cf) + { + ExpExperimentTable ret = ExperimentService.get().createExperimentTable(EXPERIMENTS_MEMBERSHIP_FOR_RUN_TABLE_NAME, this, cf); + setupTable(ret); + // Don't include exp.experiment rows that are assay batches + ret.setBatchProtocol(null); + ret.getMutableColumn(ExpExperimentTable.Column.RunCount).setHidden(true); + + ret.addExperimentMembershipColumn(run); + List defaultCols = new ArrayList<>(ret.getDefaultVisibleColumns()); + defaultCols.add(0, FieldKey.fromParts("RunMembership")); + defaultCols.remove(FieldKey.fromParts(ExpExperimentTable.Column.RunCount.name())); + ret.setDefaultVisibleColumns(defaultCols); + + return ret; + } + + // Use a holder pattern to initialize table names lazily, since ExpSchema gets referenced extremely early, before + // TableType.includeTable() might be ready to be called. e.g., ComplianceService isn't initialized yet. + private static final class TableNamesHolder + { + private static final Set tableNames; + + static + { + Set names = Arrays.stream(TableType.values()) + .filter(TableType::includeTable) + .map(TableType::toString) + .collect(Collectors.toCollection(LinkedHashSet::new)); + + names.add(DATA_CLASS_CATEGORY_TABLE); + names.add(SAMPLE_STATE_TYPE_TABLE); + names.add(SAMPLE_TYPE_CATEGORY_TABLE); + names.add(MEASUREMENT_UNITS_TABLE); + + tableNames = Collections.unmodifiableSet(names); + } + } + + public static final String SCHEMA_NAME = "exp"; + public static final String SCHEMA_DESCR = "Contains data about experiment runs, data files, materials, sample types, etc."; + + static public void register(final Module module) + { + DefaultSchema.registerProvider(SCHEMA_NAME, new DefaultSchema.SchemaProvider(module) + { + @Override + public boolean isAvailable(DefaultSchema schema, Module module) + { + return true; + } + + @Override + public QuerySchema createSchema(DefaultSchema schema, Module module) + { + return new ExpSchema(schema.getUser(), schema.getContainer()); + } + }); + } + + public SamplesSchema getSamplesSchema() + { + SamplesSchema schema = new SamplesSchema(this); + schema.setContainerFilter(_containerFilter); + return schema; + } + + public ExpSchema(User user, Container container) + { + super(SCHEMA_NAME, SCHEMA_DESCR, user, container, ExperimentService.get().getSchema()); + } + + @Override + public Set getTableNames() + { + return TableNamesHolder.tableNames; + } + + @Override + public TableInfo createTable(String name, ContainerFilter cf) + { + for (TableType tableType : TableType.values()) + { + if (tableType.name().equalsIgnoreCase(name) && tableType.includeTable()) + { + return tableType.createTable(this, tableType.name(), cf); + } + } + + if ("Experiments".equalsIgnoreCase(name)) + { + // Support "Experiments" as a legacy name for the RunGroups table + return TableType.RunGroups.createTable(this, name, cf); + } + if ("Datas".equalsIgnoreCase(name)) + { + /// Support "Datas" as a legacy name for the Data table + return TableType.Data.createTable(this, name, cf); + } + if (EXPERIMENTS_MEMBERSHIP_FOR_RUN_TABLE_NAME.equalsIgnoreCase(name)) + { + return createExperimentsTableWithRunMemberships(null, cf); + } + + if (DATA_CLASS_CATEGORY_TABLE.equalsIgnoreCase(name)) + { + return new EnumTableInfo<>(DataClassCategoryType.class, this, DataClassCategoryType::name, true, "Contains the list of available data class category types."); + } + + if (SAMPLE_STATE_TYPE_TABLE.equalsIgnoreCase(name)) + { + return new EnumTableInfo<>(SampleStateType.class, this, SampleStateType::name, true, "Contains the available sample state (status) types."); + } + + if (SAMPLE_TYPE_CATEGORY_TABLE.equalsIgnoreCase(name)) + { + return new EnumTableInfo<>(SampleTypeCategoryType.class, this, SampleTypeCategoryType::name, true, "Contains the list of available sample type category types."); + } + + if (MEASUREMENT_UNITS_TABLE.equalsIgnoreCase(name)) + { + // Create an EnumSet of the KindOfQuantity getCommonUnits + List commonUnits = KindOfQuantity.getSupportedUnits(); + EnumTableInfo table = new EnumTableInfo<>(Unit.class, EnumSet.copyOf(commonUnits), this, Unit::name, Unit::ordinal, false, "Contains the list of available units for measurements such as sample stored amounts."); + table.setPublicSchemaName(this.getName()); + table.setName(MEASUREMENT_UNITS_TABLE); + return table; + } + + return null; + } + + /** + * Exposed as EnumTableInfo. Update stateTypeEnum in qcStates.xsd if the enum values change. + */ + public enum SampleStateType + { + Available(Set.of(SampleTypeService.SampleOperations.values())), + Consumed(Set.of( + SampleTypeService.SampleOperations.EditMetadata, + SampleTypeService.SampleOperations.EditLineage, + SampleTypeService.SampleOperations.RemoveFromStorage, + SampleTypeService.SampleOperations.AddToPicklist, + SampleTypeService.SampleOperations.Delete, + SampleTypeService.SampleOperations.AddToWorkflow, + SampleTypeService.SampleOperations.RemoveFromWorkflow, + SampleTypeService.SampleOperations.AddAssayData, + SampleTypeService.SampleOperations.LinkToStudy, + SampleTypeService.SampleOperations.RecallFromStudy, + SampleTypeService.SampleOperations.Move + )), + Locked(Set.of( + SampleTypeService.SampleOperations.AddToPicklist + )); + + Set _permittedOps; + + SampleStateType(Set permittedOps) + { + _permittedOps = permittedOps; + } + + public Set getPermittedOps() + { + return _permittedOps; + } + } + + + /** + * Exposed as EnumTableInfo + * + */ + public enum DataClassCategoryType + { + registry(null), + media(null), + sources(ADDITIONAL_SOURCES_AUDIT_FIELDS); + + public final Set additionalAuditFields; + + DataClassCategoryType(@Nullable Set addlAuditFields) + { + this.additionalAuditFields = addlAuditFields; + } + + public static DataClassCategoryType fromString(String typeVal) { + for (DataClassCategoryType type : DataClassCategoryType.values()) { + if (type.name().equalsIgnoreCase(typeVal)) { + return type; + } + } + return null; + } + } + + public enum SampleTypeCategoryType + { + media; + + public static SampleTypeCategoryType fromString(String typeVal) { + for (SampleTypeCategoryType type : SampleTypeCategoryType.values()) { + if (type.name().equalsIgnoreCase(typeVal)) { + return type; + } + } + return null; + } + } + + @Override + public Set getSchemaNames() + { + if (_restricted) + return Collections.emptySet(); + + Set names = new TreeSet<>(String.CASE_INSENSITIVE_ORDER); + names.addAll(super.getSchemaNames()); + names.add(NestedSchemas.materials.name()); + names.add(NestedSchemas.data.name()); + return names; + } + + public QuerySchema getSchema(NestedSchemas schema) + { + return getSchema(schema.name()); + } + + @Override + public QuerySchema getSchema(String name) + { + if (_restricted) + return null; + + // CONSIDER: also support hidden "samples" schema ? + if (name.equals(NestedSchemas.materials.name())) + return new SamplesSchema(SchemaKey.fromParts(getName(), NestedSchemas.materials.name()), getUser(), getContainer()); + + if (name.equals(NestedSchemas.data.name())) + return new DataClassUserSchema(getContainer(), getUser()); + + return super.getSchema(name); + } + + public ExpDataTable getDatasTable() + { + return (ExpDataTable)getTable(TableType.Data); + } + + public ExpDataTable getDatasTable(boolean forWrite) + { + return (ExpDataTable)getTable(TableType.Data.toString(), null, true, forWrite); + } + + public ExpRunTable getRunsTable() + { + return (ExpRunTable)getTable(TableType.Runs.toString(), null, true, false); + } + + public ExpRunTable getRunsTable(boolean forWrite) + { + return (ExpRunTable)getTable(TableType.Runs.toString(), null, true, forWrite); + } + + + public ForeignKey getProtocolApplicationForeignKey(ContainerFilter cf) + { + return new ExperimentLookupForeignKey(null, null, ExpSchema.SCHEMA_NAME, TableType.ProtocolApplications.name(), "RowId", null) + { + @Override + public TableInfo getLookupTableInfo() + { + return getTable(TableType.ProtocolApplications, cf); + } + }; + } + + public ForeignKey getProtocolForeignKey(ContainerFilter cf, String targetColumnName) + { + return new LookupForeignKey(targetColumnName) + { + @Override + public TableInfo getLookupTableInfo() + { + return getTable(TableType.Protocols.toString(), ContainerFilter.getUnsafeEverythingFilter()); + } + }; + } + + public ForeignKey getMaterialForeignKey(ContainerFilter cf, String targetColumnName) + { + return new LookupForeignKey(targetColumnName) + { + @Override + public TableInfo getLookupTableInfo() + { + return getTable(TableType.Materials.toString(), cf); + } + + @Override + public StringExpression getURL(ColumnInfo parent) + { + return getURL(parent, true); + } + }; + } + + public ForeignKey getMaterialProtocolInputForeignKey(ContainerFilter cf) + { + return new ExperimentLookupForeignKey(null, null, ExpSchema.SCHEMA_NAME, TableType.MaterialProtocolInputs.name(), "RowId", null) + { + @Override + public TableInfo getLookupTableInfo() + { + return getTable(TableType.MaterialProtocolInputs, cf); + } + }; + } + + public ForeignKey getDataProtocolInputForeignKey(ContainerFilter cf) + { + return new ExperimentLookupForeignKey(null, null, ExpSchema.SCHEMA_NAME, TableType.DataProtocolInputs.name(), "RowId", null) + { + @Override + public TableInfo getLookupTableInfo() + { + return getTable(TableType.DataProtocolInputs, cf); + } + }; + } + + public ForeignKey getJobForeignKey() + { + return new LookupForeignKey("RowId", "RowId") + { + @Override + public TableInfo getLookupTableInfo() + { + QuerySchema pipeline = getDefaultSchema().getSchema("pipeline"); + if (null == pipeline) + return null; + return pipeline.getTable("Job", getDefaultContainerFilter()); + } + + @Override + public StringExpression getURL(ColumnInfo parent) + { + return getURL(parent, true); + } + }; + } + + @Deprecated + public ForeignKey getRunIdForeignKey() + { + return getRunIdForeignKey(null); + } + + public ForeignKey getRunIdForeignKey(ContainerFilter cf) + { + return new ExperimentLookupForeignKey(null, null, ExpSchema.SCHEMA_NAME, TableType.Runs.name(), "RowId", null) + { + @Override + public TableInfo getLookupTableInfo() + { + return getTable(TableType.Runs, cf); + } + }; + } + + @Deprecated + public ForeignKey getRunGroupIdForeignKey(final boolean includeBatches) + { + return getRunGroupIdForeignKey(null, includeBatches); + } + + /** @param includeBatches if false, then filter out run groups of type batch when doing the join */ + public ForeignKey getRunGroupIdForeignKey(ContainerFilter cf, final boolean includeBatches) + { + return new ExperimentLookupForeignKey(null, null, ExpSchema.SCHEMA_NAME, TableType.RunGroups.name(), "RowId", null) + { + @Override + public TableInfo getLookupTableInfo() + { + ContainerFilter cf = getLookupContainerFilter(); + String key = getClass().getName() + "/RunGroupIdForeignKey/" + includeBatches + "/" + cf.getCacheKey(); + // since getTable(forWrite=true) does not cache, cache this tableinfo using getCachedLookupTableInfo() + return ExpSchema.this.getCachedLookupTableInfo(key, this::createLookupTableInfo); + } + + @Override + protected ContainerFilter getLookupContainerFilter() + { + return Objects.requireNonNullElse(cf, ContainerFilter.Type.CurrentPlusProjectAndShared.create(ExpSchema.this)); + } + + private TableInfo createLookupTableInfo() + { + // CONSIDER: I wonder if this shouldn't be using UnionContainerFilter(cf, CurrentPlusProjectAndShared) + ExpExperimentTable result = (ExpExperimentTable) getTable(TableType.RunGroups.name(), getLookupContainerFilter(), true, true); + if (!includeBatches) + { + result.setBatchProtocol(null); + } + result.setLocked(true); + return result; + } + }; + } + + public ForeignKey getDataIdForeignKey(ContainerFilter cf) + { + return new ExperimentLookupForeignKey(null, null, ExpSchema.SCHEMA_NAME, TableType.Data.name(), "RowId", null) + { + @Override + public TableInfo getLookupTableInfo() + { + return getTable(TableType.Data, cf); + } + }; + } + + /** + * @param domainProperty the property on which the lookup is configured + */ + @NotNull + public ForeignKey getMaterialIdForeignKey(@Nullable ExpSampleType targetSampleType, @Nullable DomainProperty domainProperty, @Nullable ContainerFilter cfParent) + { + if (targetSampleType == null) + { + return new ExperimentLookupForeignKey(null, null, ExpSchema.SCHEMA_NAME, TableType.Materials.name(), "RowId", null) + { + @Override + public TableInfo getLookupTableInfo() + { + ContainerFilter cf = new ContainerFilter.SimpleContainerFilter(getSearchContainers(getContainer(), targetSampleType, domainProperty, getUser())); + if (null != cfParent) + cf = new UnionContainerFilter(cf, cfParent); + ExpTable result = getTable(TableType.Materials, cf); + return result; + } + }; + } + return getSamplesSchema().materialIdForeignKey(targetSampleType, domainProperty, cfParent); + } + + @NotNull + public static Set getSearchContainers(Container currentContainer, @Nullable ExpSampleType st, @Nullable DomainProperty dp, User user) + { + Set searchContainers = new LinkedHashSet<>(); + if (dp != null) + { + Lookup lookup = dp.getLookup(); + if (lookup != null && lookup.getContainer() != null) + { + Container lookupContainer = lookup.getContainer(); + if (lookupContainer.hasPermission(user, ReadPermission.class)) + { + // The property is specifically targeting a container, so look there and only there + searchContainers.add(lookup.getContainer()); + } + } + } + + if (searchContainers.isEmpty()) + { + // Default to looking in the current container + searchContainers.add(currentContainer); + if (st == null || (st.getContainer().isProject() && !currentContainer.isProject())) + { + Container c = currentContainer.getParent(); + // Recurse up the chain to the project + while (c != null && !c.isRoot()) + { + if (c.hasPermission(user, ReadPermission.class)) + { + searchContainers.add(c); + } + c = c.getParent(); + } + } + Container sharedContainer = ContainerManager.getSharedContainer(); + if (st == null || st.getContainer().equals(sharedContainer)) + { + if (sharedContainer.hasPermission(user, ReadPermission.class)) + { + searchContainers.add(ContainerManager.getSharedContainer()); + } + } + } + return searchContainers; + } + + public abstract static class ExperimentLookupForeignKey extends LookupForeignKey + { + public ExperimentLookupForeignKey(String pkColumnName) + { + super(pkColumnName); + } + + public ExperimentLookupForeignKey(ActionURL baseURL, String paramName, String schemaName, String tableName, String pkColumnName, String titleColumn) + { + super(baseURL, paramName, schemaName, tableName, pkColumnName, titleColumn); + } + + @Override + public StringExpression getURL(ColumnInfo parent) + { + return getURL(parent, true); + } + } + + @Override + public @NotNull QueryView createView(ViewContext context, @NotNull QuerySettings settings, BindException errors) + { + if (TableType.DataClasses.name().equalsIgnoreCase(settings.getQueryName())) + { + return new QueryView(this, settings, errors) + { + @Override + protected boolean canInsert() + { + TableInfo table = getTable(); + return table != null && table.hasPermission(getUser(), InsertPermission.class); + } + + @Override + public boolean showImportDataButton() + { + return false; + } + }; + } + + QueryView queryView = super.createView(context, settings, errors); + + if (TableType.Materials.name().equalsIgnoreCase(settings.getQueryName()) || + TableType.Data.name().equalsIgnoreCase(settings.getQueryName()) || + TableType.Protocols.name().equalsIgnoreCase(settings.getQueryName())) + { + // Use default delete button, but without showing the confirmation text + queryView.setShowDeleteButtonConfirmationText(false); + } + + return queryView; + } + + public enum DerivationDataScopeType + { + ChildOnly, + ParentOnly, + All + } +} diff --git a/api/src/org/labkey/api/files/FileContentService.java b/api/src/org/labkey/api/files/FileContentService.java index 9ac57fb9598..0b8b8d8019e 100644 --- a/api/src/org/labkey/api/files/FileContentService.java +++ b/api/src/org/labkey/api/files/FileContentService.java @@ -1,362 +1,362 @@ -/* - * Copyright (c) 2009-2019 LabKey Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.labkey.api.files; - -import org.jetbrains.annotations.NotNull; -import org.jetbrains.annotations.Nullable; -import org.labkey.api.attachments.AttachmentDirectory; -import org.labkey.api.data.Container; -import org.labkey.api.data.SQLFragment; -import org.labkey.api.data.TableInfo; -import org.labkey.api.exp.api.DataType; -import org.labkey.api.exp.api.ExpData; -import org.labkey.api.exp.query.ExpDataTable; -import org.labkey.api.query.QueryUpdateService; -import org.labkey.api.security.User; -import org.labkey.api.services.ServiceRegistry; -import org.labkey.api.util.FileUtil; -import org.labkey.api.webdav.WebdavResource; - -import java.io.File; -import java.net.URI; -import java.nio.file.Path; -import java.util.Collection; -import java.util.List; -import java.util.Map; - -/** - * User: klum - * Date: Dec 9, 2009 - */ -public interface FileContentService -{ - String UPLOADED_FILE_NAMESPACE_PREFIX = "UploadedFile"; - DataType UPLOADED_FILE = new DataType(UPLOADED_FILE_NAMESPACE_PREFIX); - - String FILES_LINK = "@files"; - String FILE_SETS_LINK = "@filesets"; - String PIPELINE_LINK = "@pipeline"; - String SCRIPTS_LINK = "@scripts"; - String CLOUD_LINK = "@cloud"; - String ASSAY_FILES = "@assayfiles"; - - String CLOUD_ROOT_PREFIX = "/@cloud"; - - static @Nullable FileContentService get() - { - return ServiceRegistry.get().getService(FileContentService.class); - } - - static void setInstance(FileContentService impl) - { - ServiceRegistry.get().registerService(FileContentService.class, impl); - } - - /** - * Returns a list of Container in which the path resides. - */ - @NotNull - List getContainersForFilePath(java.nio.file.Path path); - - /** - * Returns the file root of the specified container. If not explicitly defined, - * it will default to a path relative to the first parent container with an override - */ - @Nullable - File getFileRoot(@NotNull Container c); - - @Nullable - java.nio.file.Path getFileRootPath(@NotNull Container c); - - /** - * Returns the file root of the specified content type for a container - */ - @Nullable - File getFileRoot(@NotNull Container c, @NotNull ContentType type); - - @Nullable - java.nio.file.Path getFileRootPath(@NotNull Container c, @NotNull ContentType type); - - @Nullable - URI getFileRootUri(@NotNull Container c, @NotNull ContentType type, @Nullable String filePath); - - void setFileRoot(@NotNull Container c, @Nullable File root); - - void setFileRootPath(@NotNull Container c, @Nullable String root); - - void setCloudRoot(@NotNull Container c, String cloudRootName); - - boolean isCloudRoot(Container container); - - String getCloudRootName(Container c); - - void disableFileRoot(Container container); - - boolean isFileRootDisabled(Container container); - - /** - * A file root can use a default root based on a single site wide root that mirrors the folder structure of - * a project. - */ - boolean isUseDefaultRoot(Container container); - - void setIsUseDefaultRoot(Container container, boolean useDefaultRoot); - - - @NotNull - File getSiteDefaultRoot(); - - @NotNull - Path getSiteDefaultRootPath(); - - @Nullable - String getProblematicFileRootMessage(); - - void setSiteDefaultRoot(File root, User user); - - void setFileRootSetViaStartupProperty(boolean fileRootSetViaStartupProperty); - - boolean isFileRootSetViaStartupProperty(); - - /** - * Create an attachmentParent object that will allow storing files in the file system - * - * @param c Container this will be attached to - * @param name Name of the parent used in getMappedAttachmentDirectory - * @param path Path to the file. If relative is true, this is the name of a subdirectory of the directory mapped to this c - * container. If relative is false, this is a fully qualified path name - * @param relative if true, path is a relative path from the directory mapped from the container - * @return the created attachment parent - */ - AttachmentDirectory registerDirectory(Container c, String name, String path, boolean relative); - - /** - * Forget about a named directory - * - * @param c Container for this attachmentParent - * @param label Name of the parent used in registerDirectory - */ - void unregisterDirectory(Container c, String label); - - /** - * Return an AttachmentParent for files in the directory mapped to this container - * - * @param c Container in the file system - * @param createDir Create the mapped directory if it doesn't exist - * @return AttachmentParent that can be passed to other methods of this interface - */ - @Nullable - AttachmentDirectory getMappedAttachmentDirectory(Container c, boolean createDir) throws UnsetRootDirectoryException, MissingRootDirectoryException; - - @Nullable - AttachmentDirectory getMappedAttachmentDirectory(Container c, ContentType contentType, boolean createDir) throws UnsetRootDirectoryException, MissingRootDirectoryException; - - /** - * Return a named AttachmentParent for files in the directory mapped to this container - * - * @param c Container in the file system - * @return AttachmentParent that can be passed to other methods of this interface - */ - AttachmentDirectory getRegisteredDirectory(Container c, String label); - - /** - * Return a named AttachmentParent for files in the directory mapped to this container - * - * @param c Container in the file system - * @return AttachmentParent that can be passed to other methods of this interface - */ - AttachmentDirectory getRegisteredDirectoryFromEntityId(Container c, String entityId); - - /** - * Return true if the supplied string is a valid project root - * - * @param root String to use as the file path - * @return boolean - */ - boolean isValidProjectRoot(String root); - - /** - * Return all AttachmentParents for files in the directory mapped to this container - * - * @param c Container in the file system - * @return Collection of attachment directories that have previously been registered - */ - @NotNull Collection getRegisteredDirectories(Container c); - - enum ContentType { - files, - pipeline, - assay, - scripts, - assayfiles - } - - String getFolderName(ContentType type); - - FilesAdminOptions getAdminOptions(Container c); - - void setAdminOptions(Container c, FilesAdminOptions options); - - void setAdminOptions(Container c, String properties); - - /** - * Returns the default file root of the specified container. This will default to a path - * relative to the first parent container with an override - */ - File getDefaultRoot(Container c, boolean createDir); - Path getDefaultRootPath(@NotNull Container c, boolean createDir); - - class DefaultRootInfo - { - private final java.nio.file.Path _path; - private final String _prettyStr; - private final boolean _isCloud; - private final String _cloudName; - - public DefaultRootInfo(java.nio.file.Path path, String prettyStr, boolean isCloud, String cloudName) - { - _path = path; - _prettyStr = prettyStr; - _isCloud = isCloud; - _cloudName = cloudName; - } - - public java.nio.file.Path getPath() - { - return _path; - } - - public String getPrettyStr() - { - return _prettyStr; - } - - public boolean isCloud() - { - return _isCloud; - } - - public String getCloudName() - { - return _cloudName; - } - } - - DefaultRootInfo getDefaultRootInfo(Container container); - - String getDomainURI(Container c); - - String getDomainURI(Container c, FilesAdminOptions.fileConfig config); - - ExpData getDataObject(WebdavResource resource, Container c); - QueryUpdateService getFilePropsUpdateService(TableInfo tinfo, Container container); - - void moveFileRoot(File prev, File dest, @Nullable User user, @Nullable Container container); - default void moveFileRoot(Path prev, Path dest, @Nullable User user, @Nullable Container container) - { - if (!FileUtil.hasCloudScheme(prev) && !FileUtil.hasCloudScheme(dest)) - { - moveFileRoot(prev.toFile(), dest.toFile(), user, container); - } - } - - /** Notifies all registered FileListeners that a file or directory has been created */ - void fireFileCreateEvent(@NotNull File created, @Nullable User user, @Nullable Container container); - default void fireFileCreateEvent(@NotNull Path created, @Nullable User user, @Nullable Container container) - { - if (!FileUtil.hasCloudScheme(created)) - fireFileCreateEvent(created.toFile(), user, container); - } - /** - * Notifies all registered FileListeners that a file or directory has moved - * @return number of rows updated across all listeners - */ - int fireFileMoveEvent(@NotNull File src, @NotNull File dest, @Nullable User user, @Nullable Container container); - default int fireFileMoveEvent(@NotNull Path src, @NotNull Path dest, @Nullable User user, @Nullable Container container) - { - if (!FileUtil.hasCloudScheme(src) && !FileUtil.hasCloudScheme(dest)) - return fireFileMoveEvent(src.toFile(), dest.toFile(), user, container); - return 0; - } - default int fireFileMoveEvent(@NotNull Path src, @NotNull Path dest, @Nullable User user, @Nullable Container sourceContainer, @Nullable Container targetContainer) - { - return fireFileMoveEvent(src, dest, user, sourceContainer); - } - - /** Notifies all registered FileListeners that a file or directory has been replaced */ - default void fireFileReplacedEvent(@NotNull Path replaced, @Nullable User user, @Nullable Container container){} - - /** Notifies all registered FileListeners that a file or directory has been deleted */ - default void fireFileDeletedEvent(@NotNull Path deleted, @Nullable User user, @Nullable Container container){} - - /** Add a listener that will be notified when files are created or are moved */ - void addFileListener(FileListener listener); - - Map> listFiles(@NotNull Container container); - - /** - * Returns a SQLFragment for file paths that this FileListener is aware of when the user is a site admin, or empty - * results otherwise. - * The expected columns are: - *
    - *
  • Container
    • - *
  • Created
    • - *
  • CreatedBy
    • - *
  • Modified
    • - *
  • ModifiedBy
    • - *
  • FilePath
    • - *
  • SourceKey
    • - *
  • SourceName
    • - *
- */ - SQLFragment listFilesQuery(@NotNull User currentUser); - - SQLFragment listSampleFilesQuery(@NotNull User currentUser); - - void setWebfilesEnabled(boolean enabled, User user); - - /** - * Return file's virtual folder path that's relative to container's file root. Roots are matched in order of @assayfiles, @files, @pipeline and then each @filesets. - * @param dataFileUrl The data file Url of file - * @param container Container in the file system - * @return folder relative to file root - */ - String getDataFileRelativeFileRootPath(@NotNull String dataFileUrl, Container container); - - enum PathType { full, serverRelative, folderRelative } - - @Nullable - URI getWebDavUrl(@NotNull Path path, @NotNull Container container, @NotNull PathType type); - - /** - * Ensure an entry in the exp.data table exists for all files in the container's file root. - */ - void ensureFileData(@NotNull ExpDataTable table); - - /** - * Allows a module to register a directory pattern to be checked in the files webpart in order to zip the matching directory before uploading. - * @param directoryPattern DirectoryPattern - * */ - void addZiploaderPattern(DirectoryPattern directoryPattern); - - /** - * Returns a list of DirectoryPattern objects for the active modules in the given container. - * */ - List getZiploaderPatterns(Container container); - - File getMoveTargetFile(String absoluteFilePath, @NotNull Container sourceContainer, @NotNull Container targetContainer); -} +/* + * Copyright (c) 2009-2019 LabKey Corporation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.labkey.api.files; + +import org.jetbrains.annotations.NotNull; +import org.jetbrains.annotations.Nullable; +import org.labkey.api.attachments.AttachmentDirectory; +import org.labkey.api.data.Container; +import org.labkey.api.data.SQLFragment; +import org.labkey.api.data.TableInfo; +import org.labkey.api.exp.api.DataType; +import org.labkey.api.exp.api.ExpData; +import org.labkey.api.exp.query.ExpDataTable; +import org.labkey.api.query.QueryUpdateService; +import org.labkey.api.security.User; +import org.labkey.api.services.ServiceRegistry; +import org.labkey.api.util.FileUtil; +import org.labkey.api.webdav.WebdavResource; + +import java.io.File; +import java.net.URI; +import java.nio.file.Path; +import java.util.Collection; +import java.util.List; +import java.util.Map; + +/** + * User: klum + * Date: Dec 9, 2009 + */ +public interface FileContentService +{ + String UPLOADED_FILE_NAMESPACE_PREFIX = "UploadedFile"; + DataType UPLOADED_FILE = new DataType(UPLOADED_FILE_NAMESPACE_PREFIX); + + String FILES_LINK = "@files"; + String FILE_SETS_LINK = "@filesets"; + String PIPELINE_LINK = "@pipeline"; + String SCRIPTS_LINK = "@scripts"; + String CLOUD_LINK = "@cloud"; + String ASSAY_FILES = "@assayfiles"; + + String CLOUD_ROOT_PREFIX = "/@cloud"; + + static @Nullable FileContentService get() + { + return ServiceRegistry.get().getService(FileContentService.class); + } + + static void setInstance(FileContentService impl) + { + ServiceRegistry.get().registerService(FileContentService.class, impl); + } + + /** + * Returns a list of Container in which the path resides. + */ + @NotNull + List getContainersForFilePath(java.nio.file.Path path); + + /** + * Returns the file root of the specified container. If not explicitly defined, + * it will default to a path relative to the first parent container with an override + */ + @Nullable + File getFileRoot(@NotNull Container c); + + @Nullable + java.nio.file.Path getFileRootPath(@NotNull Container c); + + /** + * Returns the file root of the specified content type for a container + */ + @Nullable + File getFileRoot(@NotNull Container c, @NotNull ContentType type); + + @Nullable + java.nio.file.Path getFileRootPath(@NotNull Container c, @NotNull ContentType type); + + @Nullable + URI getFileRootUri(@NotNull Container c, @NotNull ContentType type, @Nullable String filePath); + + void setFileRoot(@NotNull Container c, @Nullable File root); + + void setFileRootPath(@NotNull Container c, @Nullable String root); + + void setCloudRoot(@NotNull Container c, String cloudRootName); + + boolean isCloudRoot(Container container); + + String getCloudRootName(Container c); + + void disableFileRoot(Container container); + + boolean isFileRootDisabled(Container container); + + /** + * A file root can use a default root based on a single site wide root that mirrors the folder structure of + * a project. + */ + boolean isUseDefaultRoot(Container container); + + void setIsUseDefaultRoot(Container container, boolean useDefaultRoot); + + + @NotNull + File getSiteDefaultRoot(); + + @NotNull + Path getSiteDefaultRootPath(); + + @Nullable + String getProblematicFileRootMessage(); + + void setSiteDefaultRoot(File root, User user); + + void setFileRootSetViaStartupProperty(boolean fileRootSetViaStartupProperty); + + boolean isFileRootSetViaStartupProperty(); + + /** + * Create an attachmentParent object that will allow storing files in the file system + * + * @param c Container this will be attached to + * @param name Name of the parent used in getMappedAttachmentDirectory + * @param path Path to the file. If relative is true, this is the name of a subdirectory of the directory mapped to this c + * container. If relative is false, this is a fully qualified path name + * @param relative if true, path is a relative path from the directory mapped from the container + * @return the created attachment parent + */ + AttachmentDirectory registerDirectory(Container c, String name, String path, boolean relative); + + /** + * Forget about a named directory + * + * @param c Container for this attachmentParent + * @param label Name of the parent used in registerDirectory + */ + void unregisterDirectory(Container c, String label); + + /** + * Return an AttachmentParent for files in the directory mapped to this container + * + * @param c Container in the file system + * @param createDir Create the mapped directory if it doesn't exist + * @return AttachmentParent that can be passed to other methods of this interface + */ + @Nullable + AttachmentDirectory getMappedAttachmentDirectory(Container c, boolean createDir) throws UnsetRootDirectoryException, MissingRootDirectoryException; + + @Nullable + AttachmentDirectory getMappedAttachmentDirectory(Container c, ContentType contentType, boolean createDir) throws UnsetRootDirectoryException, MissingRootDirectoryException; + + /** + * Return a named AttachmentParent for files in the directory mapped to this container + * + * @param c Container in the file system + * @return AttachmentParent that can be passed to other methods of this interface + */ + AttachmentDirectory getRegisteredDirectory(Container c, String label); + + /** + * Return a named AttachmentParent for files in the directory mapped to this container + * + * @param c Container in the file system + * @return AttachmentParent that can be passed to other methods of this interface + */ + AttachmentDirectory getRegisteredDirectoryFromEntityId(Container c, String entityId); + + /** + * Return true if the supplied string is a valid project root + * + * @param root String to use as the file path + * @return boolean + */ + boolean isValidProjectRoot(String root); + + /** + * Return all AttachmentParents for files in the directory mapped to this container + * + * @param c Container in the file system + * @return Collection of attachment directories that have previously been registered + */ + @NotNull Collection getRegisteredDirectories(Container c); + + enum ContentType { + files, + pipeline, + assay, + scripts, + assayfiles + } + + String getFolderName(ContentType type); + + FilesAdminOptions getAdminOptions(Container c); + + void setAdminOptions(Container c, FilesAdminOptions options); + + void setAdminOptions(Container c, String properties); + + /** + * Returns the default file root of the specified container. This will default to a path + * relative to the first parent container with an override + */ + File getDefaultRoot(Container c, boolean createDir); + Path getDefaultRootPath(@NotNull Container c, boolean createDir); + + class DefaultRootInfo + { + private final java.nio.file.Path _path; + private final String _prettyStr; + private final boolean _isCloud; + private final String _cloudName; + + public DefaultRootInfo(java.nio.file.Path path, String prettyStr, boolean isCloud, String cloudName) + { + _path = path; + _prettyStr = prettyStr; + _isCloud = isCloud; + _cloudName = cloudName; + } + + public java.nio.file.Path getPath() + { + return _path; + } + + public String getPrettyStr() + { + return _prettyStr; + } + + public boolean isCloud() + { + return _isCloud; + } + + public String getCloudName() + { + return _cloudName; + } + } + + DefaultRootInfo getDefaultRootInfo(Container container); + + String getDomainURI(Container c); + + String getDomainURI(Container c, FilesAdminOptions.fileConfig config); + + ExpData getDataObject(WebdavResource resource, Container c); + QueryUpdateService getFilePropsUpdateService(TableInfo tinfo, Container container); + + void moveFileRoot(File prev, File dest, @Nullable User user, @Nullable Container container); + default void moveFileRoot(Path prev, Path dest, @Nullable User user, @Nullable Container container) + { + if (!FileUtil.hasCloudScheme(prev) && !FileUtil.hasCloudScheme(dest)) + { + moveFileRoot(prev.toFile(), dest.toFile(), user, container); + } + } + + /** Notifies all registered FileListeners that a file or directory has been created */ + void fireFileCreateEvent(@NotNull File created, @Nullable User user, @Nullable Container container); + default void fireFileCreateEvent(@NotNull Path created, @Nullable User user, @Nullable Container container) + { + if (!FileUtil.hasCloudScheme(created)) + fireFileCreateEvent(created.toFile(), user, container); + } + /** + * Notifies all registered FileListeners that a file or directory has moved + * @return number of rows updated across all listeners + */ + int fireFileMoveEvent(@NotNull File src, @NotNull File dest, @Nullable User user, @Nullable Container container); + default int fireFileMoveEvent(@NotNull Path src, @NotNull Path dest, @Nullable User user, @Nullable Container container) + { + if (!FileUtil.hasCloudScheme(src) && !FileUtil.hasCloudScheme(dest)) + return fireFileMoveEvent(src.toFile(), dest.toFile(), user, container); + return 0; + } + default int fireFileMoveEvent(@NotNull Path src, @NotNull Path dest, @Nullable User user, @Nullable Container sourceContainer, @Nullable Container targetContainer) + { + return fireFileMoveEvent(src, dest, user, sourceContainer); + } + + /** Notifies all registered FileListeners that a file or directory has been replaced */ + default void fireFileReplacedEvent(@NotNull Path replaced, @Nullable User user, @Nullable Container container){} + + /** Notifies all registered FileListeners that a file or directory has been deleted */ + default void fireFileDeletedEvent(@NotNull Path deleted, @Nullable User user, @Nullable Container container){} + + /** Add a listener that will be notified when files are created or are moved */ + void addFileListener(FileListener listener); + + Map> listFiles(@NotNull Container container); + + /** + * Returns a SQLFragment for file paths that this FileListener is aware of when the user is a site admin, or empty + * results otherwise. + * The expected columns are: + *
    + *
  • Container
    • + *
  • Created
    • + *
  • CreatedBy
    • + *
  • Modified
    • + *
  • ModifiedBy
    • + *
  • FilePath
    • + *
  • SourceKey
    • + *
  • SourceName
    • + *
+ */ + SQLFragment listFilesQuery(@NotNull User currentUser); + + SQLFragment listSampleFilesQuery(@NotNull User currentUser); + + void setWebfilesEnabled(boolean enabled, User user); + + /** + * Return file's virtual folder path that's relative to container's file root. Roots are matched in order of @assayfiles, @files, @pipeline and then each @filesets. + * @param dataFileUrl The data file Url of file + * @param container Container in the file system + * @return folder relative to file root + */ + String getDataFileRelativeFileRootPath(@NotNull String dataFileUrl, Container container); + + enum PathType { full, serverRelative, folderRelative } + + @Nullable + URI getWebDavUrl(@NotNull Path path, @NotNull Container container, @NotNull PathType type); + + /** + * Ensure an entry in the exp.data table exists for all files in the container's file root. + */ + void ensureFileData(@NotNull ExpDataTable table); + + /** + * Allows a module to register a directory pattern to be checked in the files webpart in order to zip the matching directory before uploading. + * @param directoryPattern DirectoryPattern + * */ + void addZiploaderPattern(DirectoryPattern directoryPattern); + + /** + * Returns a list of DirectoryPattern objects for the active modules in the given container. + * */ + List getZiploaderPatterns(Container container); + + File getMoveTargetFile(String absoluteFilePath, @NotNull Container sourceContainer, @NotNull Container targetContainer); +} diff --git a/api/src/org/labkey/api/files/FileListener.java b/api/src/org/labkey/api/files/FileListener.java index 218aac7e098..3e5d5810b7c 100644 --- a/api/src/org/labkey/api/files/FileListener.java +++ b/api/src/org/labkey/api/files/FileListener.java @@ -1,102 +1,102 @@ -/* - * Copyright (c) 2013-2019 LabKey Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.labkey.api.files; - -import org.jetbrains.annotations.NotNull; -import org.jetbrains.annotations.Nullable; -import org.labkey.api.data.Container; -import org.labkey.api.data.SQLFragment; -import org.labkey.api.security.User; -import org.labkey.api.util.FileUtil; - -import java.io.File; -import java.nio.file.Path; -import java.util.Collection; - -/** - * Listener that gets notified when the server moves files or directories on the file system. Method is invoked - * once at the root level of the move, not recursively for every child file. - * User: jeckels - * Date: 11/7/12 - */ -public interface FileListener -{ - String getSourceName(); - - /** - * Called AFTER the file (or directory) has already been created on disk - * @param created newly created resource - * @param user if available, the user who initiated the create - * @param container if available, the container in which the create was initiated - */ - void fileCreated(@NotNull File created, @Nullable User user, @Nullable Container container); - default void fileCreated(@NotNull Path created, @Nullable User user, @Nullable Container container) - { - if (!FileUtil.hasCloudScheme(created)) - fileCreated(created.toFile(), user, container); - } - - /** - * Called AFTER the file (or directory) has already been moved on disk - * @param src the original file path - * @param dest the new file path - * @param user if available, the user who initiated the move - * @param container if available, the container in which the move was initiated - */ - int fileMoved(@NotNull File src, @NotNull File dest, @Nullable User user, @Nullable Container container); - default int fileMoved(@NotNull Path src, @NotNull Path dest, @Nullable User user, @Nullable Container container) - { - if (!FileUtil.hasCloudScheme(src) && !FileUtil.hasCloudScheme(dest)) - return fileMoved(src.toFile(), dest.toFile(), user, container); - return 0; - } - default int fileMoved(@NotNull Path src, @NotNull Path dest, @Nullable User user, @Nullable Container sourceContainer, @Nullable Container targetContainer) - { - return fileMoved(src, dest, user, sourceContainer); - } - - default void fileReplaced(@NotNull Path replaced, @Nullable User user, @Nullable Container container){} - - default void fileDeleted(@NotNull Path deleted, @Nullable User user, @Nullable Container container) {} - - /** - * List file paths in the database this FileListener is aware of. - * @param container If not null, list files in the given container, otherwise from all containers. - */ - Collection listFiles(@Nullable Container container); // Nobody really calls this -// public Collection listFilePaths(@Nullable Container container); - - /** - * Returns a SQLFragment for file paths that this FileListener is aware of. - * The expected columns are: - *
    - *
  • Container
    • - *
  • Created
    • - *
  • CreatedBy
    • - *
  • Modified
    • - *
  • ModifiedBy
    • - *
  • FilePath
    • - *
  • SourceKey
    • - *
  • SourceName
    • - *
- */ - SQLFragment listFilesQuery(); - - default SQLFragment listSampleFilesQuery() - { - return null; - } -} +/* + * Copyright (c) 2013-2019 LabKey Corporation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.labkey.api.files; + +import org.jetbrains.annotations.NotNull; +import org.jetbrains.annotations.Nullable; +import org.labkey.api.data.Container; +import org.labkey.api.data.SQLFragment; +import org.labkey.api.security.User; +import org.labkey.api.util.FileUtil; + +import java.io.File; +import java.nio.file.Path; +import java.util.Collection; + +/** + * Listener that gets notified when the server moves files or directories on the file system. Method is invoked + * once at the root level of the move, not recursively for every child file. + * User: jeckels + * Date: 11/7/12 + */ +public interface FileListener +{ + String getSourceName(); + + /** + * Called AFTER the file (or directory) has already been created on disk + * @param created newly created resource + * @param user if available, the user who initiated the create + * @param container if available, the container in which the create was initiated + */ + void fileCreated(@NotNull File created, @Nullable User user, @Nullable Container container); + default void fileCreated(@NotNull Path created, @Nullable User user, @Nullable Container container) + { + if (!FileUtil.hasCloudScheme(created)) + fileCreated(created.toFile(), user, container); + } + + /** + * Called AFTER the file (or directory) has already been moved on disk + * @param src the original file path + * @param dest the new file path + * @param user if available, the user who initiated the move + * @param container if available, the container in which the move was initiated + */ + int fileMoved(@NotNull File src, @NotNull File dest, @Nullable User user, @Nullable Container container); + default int fileMoved(@NotNull Path src, @NotNull Path dest, @Nullable User user, @Nullable Container container) + { + if (!FileUtil.hasCloudScheme(src) && !FileUtil.hasCloudScheme(dest)) + return fileMoved(src.toFile(), dest.toFile(), user, container); + return 0; + } + default int fileMoved(@NotNull Path src, @NotNull Path dest, @Nullable User user, @Nullable Container sourceContainer, @Nullable Container targetContainer) + { + return fileMoved(src, dest, user, sourceContainer); + } + + default void fileReplaced(@NotNull Path replaced, @Nullable User user, @Nullable Container container){} + + default void fileDeleted(@NotNull Path deleted, @Nullable User user, @Nullable Container container) {} + + /** + * List file paths in the database this FileListener is aware of. + * @param container If not null, list files in the given container, otherwise from all containers. + */ + Collection listFiles(@Nullable Container container); // Nobody really calls this +// public Collection listFilePaths(@Nullable Container container); + + /** + * Returns a SQLFragment for file paths that this FileListener is aware of. + * The expected columns are: + *
    + *
  • Container
    • + *
  • Created
    • + *
  • CreatedBy
    • + *
  • Modified
    • + *
  • ModifiedBy
    • + *
  • FilePath
    • + *
  • SourceKey
    • + *
  • SourceName
    • + *
+ */ + SQLFragment listFilesQuery(); + + default SQLFragment listSampleFilesQuery() + { + return null; + } +} diff --git a/api/src/org/labkey/api/files/TableUpdaterFileListener.java b/api/src/org/labkey/api/files/TableUpdaterFileListener.java index 2b276896bda..1e3233a82f0 100644 --- a/api/src/org/labkey/api/files/TableUpdaterFileListener.java +++ b/api/src/org/labkey/api/files/TableUpdaterFileListener.java @@ -1,461 +1,461 @@ -/* - * Copyright (c) 2013-2019 LabKey Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.labkey.api.files; - -import org.apache.commons.lang3.StringUtils; -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; -import org.jetbrains.annotations.NotNull; -import org.jetbrains.annotations.Nullable; -import org.labkey.api.data.ColumnInfo; -import org.labkey.api.data.CompareType; -import org.labkey.api.data.Container; -import org.labkey.api.data.DbSchema; -import org.labkey.api.data.SQLFragment; -import org.labkey.api.data.SimpleFilter; -import org.labkey.api.data.Sort; -import org.labkey.api.data.SqlExecutor; -import org.labkey.api.data.Table; -import org.labkey.api.data.TableInfo; -import org.labkey.api.data.TableSelector; -import org.labkey.api.data.dialect.SqlDialect; -import org.labkey.api.security.User; -import org.labkey.api.util.FileUtil; - -import java.io.File; -import java.nio.file.Files; -import java.nio.file.Path; -import java.util.Collection; -import java.util.Collections; -import java.util.Date; -import java.util.Set; - -/** - * FileListener implementation that can update tables that store file paths in various flavors (URI, standard OS - * paths, etc). - * User: jeckels - * Date: 11/7/12 - */ -public class TableUpdaterFileListener implements FileListener -{ - private static final Logger LOG = LogManager.getLogger(TableUpdaterFileListener.class); - - public static final String TABLE_ALIAS = "x"; - - private final TableInfo _table; - private final SQLFragment _containerFrag; - private final ColumnInfo _pathColumn; - private final PathGetter _pathGetter; - private final ColumnInfo _keyColumn; - - public interface PathGetter - { - /** @return the string that is expected to be in the database */ - String get(File f); - String get(Path f); - /** @return the file path separator (typically '/' or '\') */ - String getSeparatorSuffix(Path path); - } - - public enum Type implements PathGetter - { - /** Turns the file path into a "file:"-prefixed URI */ - uri - { - @Override - public String get(Path path) - { - return FileUtil.pathToString(path); - } - - @Override - public String get(File f) - { - return FileUtil.uriToString(f.toURI()); - } - - @Override - public String getSeparatorSuffix(Path path) - { - return "/"; - } - }, - - /** Just uses getPath() to turn the File into a String */ - filePath - { - @Override - public String get(Path path) - { - if (FileUtil.hasCloudScheme(path)) - return FileUtil.pathToString(path); - else - return get(path.toFile()); - } - - @Override - public String get(File f) - { - return f.getPath(); - } - - @Override - public String getSeparatorSuffix(Path path) - { - return FileUtil.hasCloudScheme(path) ? "/" : File.separator; - } - }, - - /** field has root of file path */ - fileRootPath - { - @Override - public String get(Path path) - { - return filePath.get(path); - } - - @Override - public String get(File f) - { - return filePath.get(f); - } - - @Override - public String getSeparatorSuffix(Path path) - { - return FileUtil.hasCloudScheme(path) ? "/" : File.separator; - } - }, - - /** Just uses getPath() to turn the File into a String, but replaces all backslashes with forward slashes */ - filePathForwardSlash - { - @Override - public String get(Path path) - { - if (FileUtil.hasCloudScheme(path)) - return FileUtil.pathToString(path); - else - return get(path.toFile()); - } - - @Override - public String get(File f) - { - return f.getPath().replace('\\', '/'); - } - - @Override - public String getSeparatorSuffix(Path path) - { - return "/"; - } - } - } - - public TableUpdaterFileListener(TableInfo table, String pathColumn, PathGetter pathGetter) - { - this(table, pathColumn, pathGetter, null, null); - } - - public TableUpdaterFileListener(TableInfo table, String pathColumn, PathGetter pathGetter, @Nullable String keyColumn) - { - this(table, pathColumn, pathGetter, keyColumn, null); - } - - public TableUpdaterFileListener(TableInfo table, String pathColumnName, PathGetter pathGetter, @Nullable String keyColumnName, @Nullable SQLFragment containerFrag) - { - _table = table; - _pathColumn = table.getColumn(pathColumnName); - if (null == _pathColumn) - throw new IllegalStateException("Column not found: " + pathColumnName); - _keyColumn = null == keyColumnName ? null : table.getColumn(keyColumnName); - if (null != keyColumnName && null == _keyColumn) - throw new IllegalStateException("Column not found: " + keyColumnName); - _pathGetter = pathGetter; - _containerFrag = containerFrag; - } - - @Override - public String getSourceName() - { - return _table.getSchema().getName() + "." + _table.getName() + "." + _pathColumn.getName(); - } - - public String getSourceSelect(SqlDialect sqlDialect) - { - String schema = sqlDialect.getStringHandler().quoteStringLiteral(_table.getSchema().getName()); - String table = sqlDialect.getStringHandler().quoteStringLiteral(_table.getName()); - String column = sqlDialect.getStringHandler().quoteStringLiteral(_pathColumn.getName()); - return schema + "." + table + "." + column; - } - - @Override - public void fileCreated(@NotNull File created, @Nullable User user, @Nullable Container container) - { - } - - @Override - public void fileCreated(@NotNull Path created, @Nullable User user, @Nullable Container container) - { - } - - @Override - public int fileMoved(@NotNull File src, @NotNull File dest, @Nullable User user, @Nullable Container container) - { - return fileMoved(src.toPath(), dest.toPath(), user, container); - } - - @Override - public int fileMoved(@NotNull Path src, @NotNull Path dest, @Nullable User user, @Nullable Container container) - { - DbSchema schema = _table.getSchema(); - SqlDialect dialect = schema.getSqlDialect(); - - // Build up SQL that can be used for both the file and any children - SQLFragment sharedSQL = new SQLFragment("UPDATE "); - sharedSQL.append(_table); - sharedSQL.append(_table.getSqlDialect().isSqlServer() ? " WITH (UPDLOCK)" : ""); - sharedSQL.append(" SET "); - if (_table.getColumn("Modified") != null) - { - sharedSQL.append("Modified = ?, "); - sharedSQL.add(new Date()); - } - if (_table.getColumn("ModifiedBy") != null && user != null) - { - sharedSQL.append("ModifiedBy = ?, "); - sharedSQL.add(user.getUserId()); - } - sharedSQL.appendIdentifier(_pathColumn.getSelectIdentifier()); - sharedSQL.append(" = "); - - String srcPath = getSourcePath(src, container); - String destPath = _pathGetter.get(dest); - String srcPathWithout = null; - - // If it's a directory, prep versions with and without a trailing slash. Check the dest because it's already moved by the time this fires - if (srcPath.endsWith("/") || srcPath.endsWith("\\") || Files.isDirectory(dest)) - { - srcPath = getWithSeparator(src, container); - srcPathWithout = getWithoutSeparator(srcPath); - destPath = getWithoutSeparator(destPath); - } - - // Now build up the SQL to handle this specific path - SQLFragment singleEntrySQL = new SQLFragment(sharedSQL); - singleEntrySQL.append("? WHERE ("); - singleEntrySQL.appendIdentifier(_pathColumn.getSelectIdentifier()); - singleEntrySQL.append(" = ?"); - singleEntrySQL.add(destPath); - singleEntrySQL.add(srcPath); - if (null != srcPathWithout) - { - singleEntrySQL.append(" OR "); - singleEntrySQL.appendIdentifier(_pathColumn.getSelectIdentifier()); - singleEntrySQL.append(" = ?"); - singleEntrySQL.add(srcPathWithout); - } - singleEntrySQL.append(")"); - - int rows = schema.getScope().executeWithRetry(tx -> new SqlExecutor(schema).execute(singleEntrySQL)); - LOG.info("Updated " + rows + " row in " + _table + " for move from " + src + " to " + dest); - - // Handle updating child paths, unless we know that the entry is a file. If it's not (either it's a - // directory or it doesn't exist), then try to fix up child records - if ((!Files.exists(dest) || Files.isDirectory(dest))) - { - String separatorSuffix = _pathGetter.getSeparatorSuffix(dest); - if (!srcPath.endsWith(separatorSuffix)) - { - srcPath = srcPath + separatorSuffix; - } - if (!destPath.endsWith(separatorSuffix)) - { - destPath = destPath + separatorSuffix; - } - - int childRowsUpdated = 0; - - // Consider paths with file:///... - if (srcPath.startsWith("file:")) - { - srcPath = "file://" + srcPath.replaceFirst("^file:/+", "/"); - } - SQLFragment whereClause = new SQLFragment(" WHERE "); - whereClause.append(dialect.getStringIndexOfFunction(new SQLFragment("?", srcPath), _pathColumn.getSelectIdentifier().getSql())).append(" = 1"); - - // Make the SQL to handle children - SQLFragment childPathsSQL = new SQLFragment(sharedSQL); - childPathsSQL.append(dialect.concatenate(new SQLFragment("?", destPath), new SQLFragment(dialect.getSubstringFunction(_pathColumn.getSelectIdentifier().getSql(), new SQLFragment(Integer.toString(srcPath.length() + 1)), new SQLFragment("5000"))))); - childPathsSQL.append(whereClause); - childRowsUpdated += new SqlExecutor(schema).execute(childPathsSQL); - - LOG.info("Updated " + childRowsUpdated + " child paths in " + _table + " rows for move from " + src + " to " + dest); - return childRowsUpdated; - } - return 0; - } - - @NotNull - private String getWithSeparator(Path path, Container container) - { - String result = getSourcePath(path, container); - String separator = _pathGetter.getSeparatorSuffix(path); - return result + (result.endsWith(separator) ? "" : separator); - } - - @NotNull - private String getWithoutSeparator(String path) - { - return (path.endsWith("/") || path.endsWith("\\")) ? path.substring(0, path.length() - 1): path; - } - - @Override - public Collection listFiles(@Nullable Container container) - { - Set columns = Collections.singleton(_pathColumn.getName()); - SimpleFilter filter = new SimpleFilter(); - filter.addCondition(_pathColumn, null, CompareType.NONBLANK); - if (container != null) - { - ColumnInfo containerColumn = _table.getColumn("container"); - if (containerColumn == null) - containerColumn = _table.getColumn("folder"); - - if (containerColumn != null) - filter.addCondition(containerColumn, container.getEntityId()); - else - filter.addCondition(new SimpleFilter.SQLClause("1 = 0", null)); - } - - Sort sort = new Sort(_pathColumn.getFieldKey()); - TableSelector selector = new TableSelector(_table, columns, filter, sort); - - selector.setMaxRows(Table.ALL_ROWS); - return selector.getArrayList(File.class); - } - - @Override - public SQLFragment listFilesQuery() - { - return listFilesQuery(false, null, false); - } - - public SQLFragment listFilesQuery(boolean skipCreatedModified, String filePath, boolean extractName) - { - SQLFragment selectFrag = new SQLFragment(); - selectFrag.append("SELECT\n"); - - if (_containerFrag != null) - selectFrag.append("(").append(_containerFrag).append(") AS Container,\n"); - else if (_table.getColumn("Container") != null) - selectFrag.append(" Container,\n"); - else if (_table.getColumn("Folder") != null) - selectFrag.append(" Folder AS Container,\n"); - else - selectFrag.append(" NULL AS Container,\n"); - - if (!skipCreatedModified) - { - if (_table.getColumn("Created") != null) - selectFrag.append(" Created,\n"); - else - selectFrag.append(" NULL AS Created,\n"); - - if (_table.getColumn("CreatedBy") != null) - selectFrag.append(" CreatedBy,\n"); - else - selectFrag.append(" NULL AS CreatedBy,\n"); - - if (_table.getColumn("Modified") != null) - selectFrag.append(" Modified,\n"); - else - selectFrag.append(" NULL AS Modified,\n"); - - if (_table.getColumn("ModifiedBy") != null) - selectFrag.append(" ModifiedBy,\n"); - else - selectFrag.append(" NULL AS ModifiedBy,\n"); - } - - selectFrag.append(" ").appendIdentifier(_pathColumn.getSelectIdentifier()).append(" AS FilePath,\n"); - - if (extractName) - { - SqlDialect dialect = _table.getSchema().getSqlDialect(); - SQLFragment fileNameFrag = new SQLFragment(); - fileNameFrag.append("regexp_replace(").appendIdentifier(_pathColumn.getSelectIdentifier()).append(", "); - fileNameFrag.append(dialect.getStringHandler().quoteStringLiteral(".*/")).append(", "); - fileNameFrag.append(dialect.getStringHandler().quoteStringLiteral("")).append(")");; - selectFrag.append(" ").append(fileNameFrag).append(" AS FilePathShort,\n"); - } - - if (_keyColumn != null) - selectFrag.append(" ").appendIdentifier(_keyColumn.getSelectIdentifier()).append(" AS SourceKey,\n"); - else - selectFrag.append(" NULL AS SourceKey,\n"); - - //selectFrag.append(" ? AS SourceName\n").add(getName()); - selectFrag.append(" ").appendValue(getSourceSelect(_table.getSchema().getSqlDialect())).append(" AS SourceName\n"); - - selectFrag.append("FROM ").append(_table, TABLE_ALIAS).append("\n"); - selectFrag.append("WHERE ").appendIdentifier(_pathColumn.getSelectIdentifier()); - - if (StringUtils.isEmpty(filePath)) - selectFrag.append(" IS NOT NULL\n"); - else - selectFrag.append(" = ").appendStringLiteral(filePath, _table.getSchema().getSqlDialect()).append("\n"); - - return selectFrag; - } - - @NotNull - private String getSourcePath(Path path, Container container) - { - // For uri pathGetter, check that file path exists in table, looking for legacy as well - if (Type.uri == _pathGetter) - { - String srcPath = _pathGetter.get(path); - if (pathExists(srcPath, container)) - return srcPath; - - if (!FileUtil.hasCloudScheme(path)) - { - srcPath = path.toFile().getPath(); // File path format (/users/...) - if (pathExists(srcPath, container)) - return srcPath; - } - - // use original if none found, for directories - } - - return _pathGetter.get(path); - } - - private boolean pathExists(String srcPath, Container container) - { - - SimpleFilter filter = (null != _table.getColumn("container")) ? - SimpleFilter.createContainerFilter(container) : - (null != _table.getColumn("folder")) ? - SimpleFilter.createContainerFilter(container, "folder") : - new SimpleFilter(); - filter.addCondition(_pathColumn, srcPath); - return new TableSelector(_table, filter, null).exists(); - } -} +/* + * Copyright (c) 2013-2019 LabKey Corporation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.labkey.api.files; + +import org.apache.commons.lang3.StringUtils; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.jetbrains.annotations.NotNull; +import org.jetbrains.annotations.Nullable; +import org.labkey.api.data.ColumnInfo; +import org.labkey.api.data.CompareType; +import org.labkey.api.data.Container; +import org.labkey.api.data.DbSchema; +import org.labkey.api.data.SQLFragment; +import org.labkey.api.data.SimpleFilter; +import org.labkey.api.data.Sort; +import org.labkey.api.data.SqlExecutor; +import org.labkey.api.data.Table; +import org.labkey.api.data.TableInfo; +import org.labkey.api.data.TableSelector; +import org.labkey.api.data.dialect.SqlDialect; +import org.labkey.api.security.User; +import org.labkey.api.util.FileUtil; + +import java.io.File; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.Collection; +import java.util.Collections; +import java.util.Date; +import java.util.Set; + +/** + * FileListener implementation that can update tables that store file paths in various flavors (URI, standard OS + * paths, etc). + * User: jeckels + * Date: 11/7/12 + */ +public class TableUpdaterFileListener implements FileListener +{ + private static final Logger LOG = LogManager.getLogger(TableUpdaterFileListener.class); + + public static final String TABLE_ALIAS = "x"; + + private final TableInfo _table; + private final SQLFragment _containerFrag; + private final ColumnInfo _pathColumn; + private final PathGetter _pathGetter; + private final ColumnInfo _keyColumn; + + public interface PathGetter + { + /** @return the string that is expected to be in the database */ + String get(File f); + String get(Path f); + /** @return the file path separator (typically '/' or '\') */ + String getSeparatorSuffix(Path path); + } + + public enum Type implements PathGetter + { + /** Turns the file path into a "file:"-prefixed URI */ + uri + { + @Override + public String get(Path path) + { + return FileUtil.pathToString(path); + } + + @Override + public String get(File f) + { + return FileUtil.uriToString(f.toURI()); + } + + @Override + public String getSeparatorSuffix(Path path) + { + return "/"; + } + }, + + /** Just uses getPath() to turn the File into a String */ + filePath + { + @Override + public String get(Path path) + { + if (FileUtil.hasCloudScheme(path)) + return FileUtil.pathToString(path); + else + return get(path.toFile()); + } + + @Override + public String get(File f) + { + return f.getPath(); + } + + @Override + public String getSeparatorSuffix(Path path) + { + return FileUtil.hasCloudScheme(path) ? "/" : File.separator; + } + }, + + /** field has root of file path */ + fileRootPath + { + @Override + public String get(Path path) + { + return filePath.get(path); + } + + @Override + public String get(File f) + { + return filePath.get(f); + } + + @Override + public String getSeparatorSuffix(Path path) + { + return FileUtil.hasCloudScheme(path) ? "/" : File.separator; + } + }, + + /** Just uses getPath() to turn the File into a String, but replaces all backslashes with forward slashes */ + filePathForwardSlash + { + @Override + public String get(Path path) + { + if (FileUtil.hasCloudScheme(path)) + return FileUtil.pathToString(path); + else + return get(path.toFile()); + } + + @Override + public String get(File f) + { + return f.getPath().replace('\\', '/'); + } + + @Override + public String getSeparatorSuffix(Path path) + { + return "/"; + } + } + } + + public TableUpdaterFileListener(TableInfo table, String pathColumn, PathGetter pathGetter) + { + this(table, pathColumn, pathGetter, null, null); + } + + public TableUpdaterFileListener(TableInfo table, String pathColumn, PathGetter pathGetter, @Nullable String keyColumn) + { + this(table, pathColumn, pathGetter, keyColumn, null); + } + + public TableUpdaterFileListener(TableInfo table, String pathColumnName, PathGetter pathGetter, @Nullable String keyColumnName, @Nullable SQLFragment containerFrag) + { + _table = table; + _pathColumn = table.getColumn(pathColumnName); + if (null == _pathColumn) + throw new IllegalStateException("Column not found: " + pathColumnName); + _keyColumn = null == keyColumnName ? null : table.getColumn(keyColumnName); + if (null != keyColumnName && null == _keyColumn) + throw new IllegalStateException("Column not found: " + keyColumnName); + _pathGetter = pathGetter; + _containerFrag = containerFrag; + } + + @Override + public String getSourceName() + { + return _table.getSchema().getName() + "." + _table.getName() + "." + _pathColumn.getName(); + } + + public String getSourceSelect(SqlDialect sqlDialect) + { + String schema = sqlDialect.getStringHandler().quoteStringLiteral(_table.getSchema().getName()); + String table = sqlDialect.getStringHandler().quoteStringLiteral(_table.getName()); + String column = sqlDialect.getStringHandler().quoteStringLiteral(_pathColumn.getName()); + return schema + "." + table + "." + column; + } + + @Override + public void fileCreated(@NotNull File created, @Nullable User user, @Nullable Container container) + { + } + + @Override + public void fileCreated(@NotNull Path created, @Nullable User user, @Nullable Container container) + { + } + + @Override + public int fileMoved(@NotNull File src, @NotNull File dest, @Nullable User user, @Nullable Container container) + { + return fileMoved(src.toPath(), dest.toPath(), user, container); + } + + @Override + public int fileMoved(@NotNull Path src, @NotNull Path dest, @Nullable User user, @Nullable Container container) + { + DbSchema schema = _table.getSchema(); + SqlDialect dialect = schema.getSqlDialect(); + + // Build up SQL that can be used for both the file and any children + SQLFragment sharedSQL = new SQLFragment("UPDATE "); + sharedSQL.append(_table); + sharedSQL.append(_table.getSqlDialect().isSqlServer() ? " WITH (UPDLOCK)" : ""); + sharedSQL.append(" SET "); + if (_table.getColumn("Modified") != null) + { + sharedSQL.append("Modified = ?, "); + sharedSQL.add(new Date()); + } + if (_table.getColumn("ModifiedBy") != null && user != null) + { + sharedSQL.append("ModifiedBy = ?, "); + sharedSQL.add(user.getUserId()); + } + sharedSQL.appendIdentifier(_pathColumn.getSelectIdentifier()); + sharedSQL.append(" = "); + + String srcPath = getSourcePath(src, container); + String destPath = _pathGetter.get(dest); + String srcPathWithout = null; + + // If it's a directory, prep versions with and without a trailing slash. Check the dest because it's already moved by the time this fires + if (srcPath.endsWith("/") || srcPath.endsWith("\\") || Files.isDirectory(dest)) + { + srcPath = getWithSeparator(src, container); + srcPathWithout = getWithoutSeparator(srcPath); + destPath = getWithoutSeparator(destPath); + } + + // Now build up the SQL to handle this specific path + SQLFragment singleEntrySQL = new SQLFragment(sharedSQL); + singleEntrySQL.append("? WHERE ("); + singleEntrySQL.appendIdentifier(_pathColumn.getSelectIdentifier()); + singleEntrySQL.append(" = ?"); + singleEntrySQL.add(destPath); + singleEntrySQL.add(srcPath); + if (null != srcPathWithout) + { + singleEntrySQL.append(" OR "); + singleEntrySQL.appendIdentifier(_pathColumn.getSelectIdentifier()); + singleEntrySQL.append(" = ?"); + singleEntrySQL.add(srcPathWithout); + } + singleEntrySQL.append(")"); + + int rows = schema.getScope().executeWithRetry(tx -> new SqlExecutor(schema).execute(singleEntrySQL)); + LOG.info("Updated " + rows + " row in " + _table + " for move from " + src + " to " + dest); + + // Handle updating child paths, unless we know that the entry is a file. If it's not (either it's a + // directory or it doesn't exist), then try to fix up child records + if ((!Files.exists(dest) || Files.isDirectory(dest))) + { + String separatorSuffix = _pathGetter.getSeparatorSuffix(dest); + if (!srcPath.endsWith(separatorSuffix)) + { + srcPath = srcPath + separatorSuffix; + } + if (!destPath.endsWith(separatorSuffix)) + { + destPath = destPath + separatorSuffix; + } + + int childRowsUpdated = 0; + + // Consider paths with file:///... + if (srcPath.startsWith("file:")) + { + srcPath = "file://" + srcPath.replaceFirst("^file:/+", "/"); + } + SQLFragment whereClause = new SQLFragment(" WHERE "); + whereClause.append(dialect.getStringIndexOfFunction(new SQLFragment("?", srcPath), _pathColumn.getSelectIdentifier().getSql())).append(" = 1"); + + // Make the SQL to handle children + SQLFragment childPathsSQL = new SQLFragment(sharedSQL); + childPathsSQL.append(dialect.concatenate(new SQLFragment("?", destPath), new SQLFragment(dialect.getSubstringFunction(_pathColumn.getSelectIdentifier().getSql(), new SQLFragment(Integer.toString(srcPath.length() + 1)), new SQLFragment("5000"))))); + childPathsSQL.append(whereClause); + childRowsUpdated += new SqlExecutor(schema).execute(childPathsSQL); + + LOG.info("Updated " + childRowsUpdated + " child paths in " + _table + " rows for move from " + src + " to " + dest); + return childRowsUpdated; + } + return 0; + } + + @NotNull + private String getWithSeparator(Path path, Container container) + { + String result = getSourcePath(path, container); + String separator = _pathGetter.getSeparatorSuffix(path); + return result + (result.endsWith(separator) ? "" : separator); + } + + @NotNull + private String getWithoutSeparator(String path) + { + return (path.endsWith("/") || path.endsWith("\\")) ? path.substring(0, path.length() - 1): path; + } + + @Override + public Collection listFiles(@Nullable Container container) + { + Set columns = Collections.singleton(_pathColumn.getName()); + SimpleFilter filter = new SimpleFilter(); + filter.addCondition(_pathColumn, null, CompareType.NONBLANK); + if (container != null) + { + ColumnInfo containerColumn = _table.getColumn("container"); + if (containerColumn == null) + containerColumn = _table.getColumn("folder"); + + if (containerColumn != null) + filter.addCondition(containerColumn, container.getEntityId()); + else + filter.addCondition(new SimpleFilter.SQLClause("1 = 0", null)); + } + + Sort sort = new Sort(_pathColumn.getFieldKey()); + TableSelector selector = new TableSelector(_table, columns, filter, sort); + + selector.setMaxRows(Table.ALL_ROWS); + return selector.getArrayList(File.class); + } + + @Override + public SQLFragment listFilesQuery() + { + return listFilesQuery(false, null, false); + } + + public SQLFragment listFilesQuery(boolean skipCreatedModified, String filePath, boolean extractName) + { + SQLFragment selectFrag = new SQLFragment(); + selectFrag.append("SELECT\n"); + + if (_containerFrag != null) + selectFrag.append("(").append(_containerFrag).append(") AS Container,\n"); + else if (_table.getColumn("Container") != null) + selectFrag.append(" Container,\n"); + else if (_table.getColumn("Folder") != null) + selectFrag.append(" Folder AS Container,\n"); + else + selectFrag.append(" NULL AS Container,\n"); + + if (!skipCreatedModified) + { + if (_table.getColumn("Created") != null) + selectFrag.append(" Created,\n"); + else + selectFrag.append(" NULL AS Created,\n"); + + if (_table.getColumn("CreatedBy") != null) + selectFrag.append(" CreatedBy,\n"); + else + selectFrag.append(" NULL AS CreatedBy,\n"); + + if (_table.getColumn("Modified") != null) + selectFrag.append(" Modified,\n"); + else + selectFrag.append(" NULL AS Modified,\n"); + + if (_table.getColumn("ModifiedBy") != null) + selectFrag.append(" ModifiedBy,\n"); + else + selectFrag.append(" NULL AS ModifiedBy,\n"); + } + + selectFrag.append(" ").appendIdentifier(_pathColumn.getSelectIdentifier()).append(" AS FilePath,\n"); + + if (extractName) + { + SqlDialect dialect = _table.getSchema().getSqlDialect(); + SQLFragment fileNameFrag = new SQLFragment(); + fileNameFrag.append("regexp_replace(").appendIdentifier(_pathColumn.getSelectIdentifier()).append(", "); + fileNameFrag.append(dialect.getStringHandler().quoteStringLiteral(".*/")).append(", "); + fileNameFrag.append(dialect.getStringHandler().quoteStringLiteral("")).append(")");; + selectFrag.append(" ").append(fileNameFrag).append(" AS FilePathShort,\n"); + } + + if (_keyColumn != null) + selectFrag.append(" ").appendIdentifier(_keyColumn.getSelectIdentifier()).append(" AS SourceKey,\n"); + else + selectFrag.append(" NULL AS SourceKey,\n"); + + //selectFrag.append(" ? AS SourceName\n").add(getName()); + selectFrag.append(" ").appendValue(getSourceSelect(_table.getSchema().getSqlDialect())).append(" AS SourceName\n"); + + selectFrag.append("FROM ").append(_table, TABLE_ALIAS).append("\n"); + selectFrag.append("WHERE ").appendIdentifier(_pathColumn.getSelectIdentifier()); + + if (StringUtils.isEmpty(filePath)) + selectFrag.append(" IS NOT NULL\n"); + else + selectFrag.append(" = ").appendStringLiteral(filePath, _table.getSchema().getSqlDialect()).append("\n"); + + return selectFrag; + } + + @NotNull + private String getSourcePath(Path path, Container container) + { + // For uri pathGetter, check that file path exists in table, looking for legacy as well + if (Type.uri == _pathGetter) + { + String srcPath = _pathGetter.get(path); + if (pathExists(srcPath, container)) + return srcPath; + + if (!FileUtil.hasCloudScheme(path)) + { + srcPath = path.toFile().getPath(); // File path format (/users/...) + if (pathExists(srcPath, container)) + return srcPath; + } + + // use original if none found, for directories + } + + return _pathGetter.get(path); + } + + private boolean pathExists(String srcPath, Container container) + { + + SimpleFilter filter = (null != _table.getColumn("container")) ? + SimpleFilter.createContainerFilter(container) : + (null != _table.getColumn("folder")) ? + SimpleFilter.createContainerFilter(container, "folder") : + new SimpleFilter(); + filter.addCondition(_pathColumn, srcPath); + return new TableSelector(_table, filter, null).exists(); + } +} diff --git a/experiment/src/org/labkey/experiment/FileLinkFileListener.java b/experiment/src/org/labkey/experiment/FileLinkFileListener.java index 8c0d1c5b729..d0b538ef145 100644 --- a/experiment/src/org/labkey/experiment/FileLinkFileListener.java +++ b/experiment/src/org/labkey/experiment/FileLinkFileListener.java @@ -1,338 +1,338 @@ -/* - * Copyright (c) 2013-2019 LabKey Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.labkey.experiment; - -import org.apache.commons.lang3.StringUtils; -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; -import org.jetbrains.annotations.NotNull; -import org.jetbrains.annotations.Nullable; -import org.labkey.api.data.ColumnInfo; -import org.labkey.api.data.Container; -import org.labkey.api.data.DbSchema; -import org.labkey.api.data.DbSchemaType; -import org.labkey.api.data.SQLFragment; -import org.labkey.api.data.SqlExecutor; -import org.labkey.api.data.SqlSelector; -import org.labkey.api.data.TableInfo; -import org.labkey.api.data.dialect.SqlDialect; -import org.labkey.api.exp.OntologyManager; -import org.labkey.api.exp.PropertyType; -import org.labkey.api.exp.api.StorageProvisioner; -import org.labkey.api.exp.property.Domain; -import org.labkey.api.exp.property.PropertyService; -import org.labkey.api.files.FileListener; -import org.labkey.api.files.TableUpdaterFileListener; -import org.labkey.api.security.User; -import org.labkey.api.util.FileUtil; -import org.labkey.experiment.api.ExpMaterialTableImpl; -import org.labkey.experiment.api.SampleTypeServiceImpl; - -import java.io.File; -import java.nio.file.Files; -import java.nio.file.Path; -import java.util.ArrayList; -import java.util.Collection; -import java.util.concurrent.atomic.AtomicInteger; - -import static org.labkey.api.exp.api.SampleTypeDomainKind.PROVISIONED_SCHEMA_NAME; - -/** - * Handles fixup of paths stored in OntologyManager for File Link fields. These values are persisted as absolute paths - * on the server's file system. - * User: jeckels - * Date: 11/8/12 - */ -public class FileLinkFileListener implements FileListener -{ - private static final Logger LOG = LogManager.getLogger(FileLinkFileListener.class); - - @Override - public String getSourceName() - { - return "FileLinkFileListener"; - } - - @Override - public void fileCreated(@NotNull File created, @Nullable User user, @Nullable Container container) - { - } - - @Override - public void fileCreated(@NotNull Path created, @Nullable User user, @Nullable Container container) - { - } - - @Override - public int fileMoved(@NotNull File srcFile, @NotNull File destFile, @Nullable User user, @Nullable Container container) - { - return fileMoved(srcFile.toPath(), destFile.toPath(), user, container); - } - - @Override - public int fileMoved(@NotNull Path srcFile, @NotNull Path destFile, @Nullable User user, @Nullable Container container) - { - int result = updateObjectProperty(srcFile, destFile); - - result += updateHardTables(srcFile, destFile, user, container); - return result; - } - - /** Migrate FileLink values stored in exp.ObjectProperty */ - private int updateObjectProperty(Path srcFile, Path destFile) - { - String srcPath = FileUtil.hasCloudScheme(srcFile) ? FileUtil.pathToString(srcFile) : srcFile.toFile().getPath(); - String destPath = FileUtil.hasCloudScheme(destFile) ? FileUtil.pathToString(destFile) : destFile.toFile().getPath(); - - SqlDialect dialect = OntologyManager.getSqlDialect(); - - // Build up SQL that can be used for both the file and any children - SQLFragment sharedSQL = new SQLFragment("UPDATE "); - sharedSQL.append(OntologyManager.getTinfoObjectProperty()); - sharedSQL.append(" SET StringValue = "); - - SQLFragment standardWhereSQL = new SQLFragment(" WHERE PropertyId IN (SELECT PropertyId FROM "); - standardWhereSQL.append(OntologyManager.getTinfoPropertyDescriptor(), "pd"); - standardWhereSQL.append(" WHERE RangeURI = ?)"); - standardWhereSQL.add(PropertyType.FILE_LINK.getTypeUri()); - - // Now build up the SQL to handle this specific path - SQLFragment singleEntrySQL = new SQLFragment(sharedSQL); - singleEntrySQL.append("? "); - singleEntrySQL.add(destPath); - singleEntrySQL.append(standardWhereSQL); - singleEntrySQL.append(" AND StringValue = ?"); - singleEntrySQL.add(srcPath); - - int rows = new SqlExecutor(OntologyManager.getExpSchema()).execute(singleEntrySQL); - LOG.info("Updated " + rows + " row in exp.ObjectProperty for move from " + srcFile + " to " + destFile); - if (rows > 0) - { - // Clear potential object values - OntologyManager.clearPropertyCache(); - } - - // Skip attempting to fix up child paths if we know that the entry is a file. If it's not (either it's a - // directory or it doesn't exist), then try to fix up child records - if (Files.isDirectory(destFile)) - { - if (!srcPath.endsWith(File.separator)) - { - srcPath = srcPath + File.separator; - } - if (!destPath.endsWith(File.separator)) - { - destPath = destPath + File.separator; - } - - // Make the SQL to handle children - SQLFragment childPathsSQL = new SQLFragment(sharedSQL); - childPathsSQL.append(dialect.concatenate(new SQLFragment("?", destPath), new SQLFragment(dialect.getSubstringFunction("StringValue", Integer.toString(srcPath.length() + 1), "5000")))); - childPathsSQL.append(standardWhereSQL); - childPathsSQL.append(" AND "); - childPathsSQL.append(dialect.getStringIndexOfFunction(new SQLFragment("?", srcPath), new SQLFragment("StringValue"))); - childPathsSQL.append(" = 1"); - - int childRows = new SqlExecutor(OntologyManager.getExpSchema()).execute(childPathsSQL); - if (childRows > 0) - { - // Clear potential object values - OntologyManager.clearPropertyCache(); - } - LOG.info("Updated " + childRows + " child paths in exp.ObjectProperty rows for move from " + srcFile + " to " + destFile); - return rows + childRows; - } - return rows; - } - - private int updateHardTables(final Path srcFile, final Path destFile, final User user, final Container container) - { - AtomicInteger result = new AtomicInteger(0); - hardTableFileLinkColumns((schema, table, pathColumn, containerId, domainUri) -> { - // Migrate any paths that match this file move - TableUpdaterFileListener updater = new TableUpdaterFileListener(table, pathColumn.getColumnName(), TableUpdaterFileListener.Type.filePath); - int movedCount = updater.fileMoved(srcFile, destFile, user, container); - result.addAndGet(movedCount); - - if (movedCount > 0 && schema.getName().equalsIgnoreCase(PROVISIONED_SCHEMA_NAME)) - ExpMaterialTableImpl.refreshMaterializedView(domainUri, SampleTypeServiceImpl.SampleChangeType.update); - }); - return result.intValue(); - } - - private interface ForEachFileLinkColumn - { - void exec(DbSchema schema, TableInfo table, ColumnInfo pathColumn, String containerId, @Nullable String domainUri); - } - - private void hardTableFileLinkColumns(final ForEachFileLinkColumn block) - { - // Figure out all of the FileLink columns in hard tables managed by OntologyManager - SQLFragment sql = new SQLFragment("SELECT dd.Container, dd.DomainId, dd.StorageTableName, dd.StorageSchemaName, pd.Name, pd.StorageColumnName FROM "); - sql.append(OntologyManager.getTinfoDomainDescriptor(), "dd"); - sql.append(", "); - sql.append(OntologyManager.getTinfoPropertyDescriptor(), "pd"); - sql.append(", "); - sql.append(OntologyManager.getTinfoPropertyDomain(), "m"); - sql.append(" WHERE dd.DomainId = m.DomainId AND pd.PropertyId = m.PropertyId AND pd.RangeURI = ? "); - sql.add(PropertyType.FILE_LINK.getTypeUri()); - sql.append(" AND dd.StorageTableName IS NOT NULL AND dd.StorageSchemaName IS NOT NULL AND pd.Name IS NOT NULL"); - - new SqlSelector(OntologyManager.getExpSchema(), sql).forEachMap(row -> { - // Find the DbSchema/TableInfo/ColumnInfo for the FileLink column - String storageSchemaName = row.get("StorageSchemaName").toString(); - DbSchema schema = DbSchema.get(storageSchemaName, DbSchemaType.Provisioned); - Domain domain = PropertyService.get().getDomain((Integer) row.get("DomainId")); - // Issue 50781: LKSM: File fields in Sample Types sometimes showing as Text Fields - // Don't use schema.getTable(storageTableName); - if (domain != null) - { - TableInfo tableInfo = StorageProvisioner.get().getSchemaTableInfo(domain); - if (tableInfo != null) - { - String containerId = row.get("Container").toString(); - if (containerId != null) - { - ColumnInfo pathCol = tableInfo.getColumn(row.get("Name").toString()); - // Issue 53502: also try to get tableInfo column by StorageColumnName if not found by Name - if (pathCol == null) - pathCol = tableInfo.getColumn(row.get("StorageColumnName").toString()); - - if (pathCol != null) - block.exec(schema, tableInfo, pathCol, containerId, domain.getTypeURI()); - } - } - } - }); - } - - @Override - public Collection listFiles(@Nullable Container container) - { - Collection files = new ArrayList<>(); - - files.addAll(listObjectPropertyFiles(container)); - files.addAll(listHardTableFiles(container)); - - return files; - } - - private Collection listObjectPropertyFiles(@Nullable Container container) - { - SQLFragment frag = new SQLFragment(); - frag.append("SELECT StringValue\n"); - frag.append("FROM\n"); - frag.append(" ").append(OntologyManager.getTinfoObjectProperty(), "op").append(",\n"); - frag.append(" ").append(OntologyManager.getTinfoObject(), "o").append("\n"); - frag.append("WHERE "); - frag.append(" o.ObjectId = op.ObjectId AND\n"); - if (container != null) - { - frag.append(" o.Container = ? AND \n"); - frag.add(container); - } - frag.append(" PropertyId IN (\n"); - frag.append(" SELECT PropertyId\n"); - frag.append(" FROM ").append(OntologyManager.getTinfoPropertyDescriptor(), "pd").append("\n"); - frag.append(" WHERE RangeURI = ?\n").add(PropertyType.FILE_LINK.getTypeUri()); - frag.append(" )\n"); - - SqlSelector selector = new SqlSelector(OntologyManager.getExpSchema(), frag); - return selector.getArrayList(File.class); - } - - private Collection listHardTableFiles(@NotNull final Container container) - { - final Collection files = new ArrayList<>(); - hardTableFileLinkColumns((schema, table, pathColumn, containerId, domainUri) -> { - TableUpdaterFileListener updater = new TableUpdaterFileListener(table, pathColumn.getColumnName(), TableUpdaterFileListener.Type.filePath); - files.addAll(updater.listFiles(container)); - }); - return files; - } - - @Override - public SQLFragment listFilesQuery() - { - return listFilesQuery(false); - } - - public SQLFragment listFilesQuery(boolean skipCreatedModified) - { - return listFilesQuery(skipCreatedModified, null); - } - - public SQLFragment listFilesQuery(boolean skipCreatedModified, String filePath) - { - final SQLFragment frag = new SQLFragment(); - - // Object property files - frag.append("SELECT\n"); - frag.append(" o.Container,\n"); - if (!skipCreatedModified) - { - frag.append(" NULL AS Created,\n"); - frag.append(" NULL AS CreatedBy,\n"); - frag.append(" NULL AS Modified,\n"); - frag.append(" NULL AS ModifiedBy,\n"); - } - frag.append(" op.StringValue AS FilePath,\n"); - frag.append(" o.ObjectId AS SourceKey,\n"); - frag.append(" 'exp.object' AS SourceName\n"); - frag.append("FROM\n"); - frag.append(" ").append(OntologyManager.getTinfoObjectProperty(), "op").append(",\n"); - frag.append(" ").append(OntologyManager.getTinfoObject(), "o").append("\n"); - frag.append("WHERE\n"); - if (StringUtils.isEmpty(filePath)) - frag.append(" op.StringValue IS NOT NULL AND\n"); - else - frag.append(" op.StringValue = ").appendStringLiteral(filePath, OntologyManager.getTinfoObject().getSqlDialect()).append(" AND\n"); - frag.append(" o.ObjectId = op.ObjectId AND\n"); - frag.append(" PropertyId IN (\n"); - frag.append(" SELECT PropertyId\n"); - frag.append(" FROM ").append(OntologyManager.getTinfoPropertyDescriptor(), "pd").append("\n"); - frag.append(" WHERE RangeURI = ?\n").add(PropertyType.FILE_LINK.getTypeUri()); - frag.append(" )\n"); - - hardTableFileLinkColumns((schema, table, pathColumn, containerId, domainUri) -> { - SQLFragment containerFrag = new SQLFragment("?", containerId); - TableUpdaterFileListener updater = new TableUpdaterFileListener(table, pathColumn.getColumnName(), TableUpdaterFileListener.Type.filePath, null, containerFrag); - frag.append("UNION").append(StringUtils.isEmpty(filePath) ? "" : " ALL" /*keep duplicate*/).append("\n"); - frag.append(updater.listFilesQuery(skipCreatedModified, filePath, false)); - }); - - return frag; - } - - @Override - public SQLFragment listSampleFilesQuery() - { - final SQLFragment frag = new SQLFragment(); - - hardTableFileLinkColumns((schema, table, pathColumn, containerId, domainUri) -> { - if (schema.getName().equals("expsampleset")) - { - SQLFragment containerFrag = new SQLFragment("?", containerId); - TableUpdaterFileListener updater = new TableUpdaterFileListener(table, pathColumn.getColumnName(), TableUpdaterFileListener.Type.filePath, "rowid", containerFrag); - if (!frag.isEmpty()) - frag.append("UNION").append("").append("\n"); - frag.append(updater.listFilesQuery(true, null, true)); - } - }); - - return frag; - } -} +/* + * Copyright (c) 2013-2019 LabKey Corporation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.labkey.experiment; + +import org.apache.commons.lang3.StringUtils; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.jetbrains.annotations.NotNull; +import org.jetbrains.annotations.Nullable; +import org.labkey.api.data.ColumnInfo; +import org.labkey.api.data.Container; +import org.labkey.api.data.DbSchema; +import org.labkey.api.data.DbSchemaType; +import org.labkey.api.data.SQLFragment; +import org.labkey.api.data.SqlExecutor; +import org.labkey.api.data.SqlSelector; +import org.labkey.api.data.TableInfo; +import org.labkey.api.data.dialect.SqlDialect; +import org.labkey.api.exp.OntologyManager; +import org.labkey.api.exp.PropertyType; +import org.labkey.api.exp.api.StorageProvisioner; +import org.labkey.api.exp.property.Domain; +import org.labkey.api.exp.property.PropertyService; +import org.labkey.api.files.FileListener; +import org.labkey.api.files.TableUpdaterFileListener; +import org.labkey.api.security.User; +import org.labkey.api.util.FileUtil; +import org.labkey.experiment.api.ExpMaterialTableImpl; +import org.labkey.experiment.api.SampleTypeServiceImpl; + +import java.io.File; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.Collection; +import java.util.concurrent.atomic.AtomicInteger; + +import static org.labkey.api.exp.api.SampleTypeDomainKind.PROVISIONED_SCHEMA_NAME; + +/** + * Handles fixup of paths stored in OntologyManager for File Link fields. These values are persisted as absolute paths + * on the server's file system. + * User: jeckels + * Date: 11/8/12 + */ +public class FileLinkFileListener implements FileListener +{ + private static final Logger LOG = LogManager.getLogger(FileLinkFileListener.class); + + @Override + public String getSourceName() + { + return "FileLinkFileListener"; + } + + @Override + public void fileCreated(@NotNull File created, @Nullable User user, @Nullable Container container) + { + } + + @Override + public void fileCreated(@NotNull Path created, @Nullable User user, @Nullable Container container) + { + } + + @Override + public int fileMoved(@NotNull File srcFile, @NotNull File destFile, @Nullable User user, @Nullable Container container) + { + return fileMoved(srcFile.toPath(), destFile.toPath(), user, container); + } + + @Override + public int fileMoved(@NotNull Path srcFile, @NotNull Path destFile, @Nullable User user, @Nullable Container container) + { + int result = updateObjectProperty(srcFile, destFile); + + result += updateHardTables(srcFile, destFile, user, container); + return result; + } + + /** Migrate FileLink values stored in exp.ObjectProperty */ + private int updateObjectProperty(Path srcFile, Path destFile) + { + String srcPath = FileUtil.hasCloudScheme(srcFile) ? FileUtil.pathToString(srcFile) : srcFile.toFile().getPath(); + String destPath = FileUtil.hasCloudScheme(destFile) ? FileUtil.pathToString(destFile) : destFile.toFile().getPath(); + + SqlDialect dialect = OntologyManager.getSqlDialect(); + + // Build up SQL that can be used for both the file and any children + SQLFragment sharedSQL = new SQLFragment("UPDATE "); + sharedSQL.append(OntologyManager.getTinfoObjectProperty()); + sharedSQL.append(" SET StringValue = "); + + SQLFragment standardWhereSQL = new SQLFragment(" WHERE PropertyId IN (SELECT PropertyId FROM "); + standardWhereSQL.append(OntologyManager.getTinfoPropertyDescriptor(), "pd"); + standardWhereSQL.append(" WHERE RangeURI = ?)"); + standardWhereSQL.add(PropertyType.FILE_LINK.getTypeUri()); + + // Now build up the SQL to handle this specific path + SQLFragment singleEntrySQL = new SQLFragment(sharedSQL); + singleEntrySQL.append("? "); + singleEntrySQL.add(destPath); + singleEntrySQL.append(standardWhereSQL); + singleEntrySQL.append(" AND StringValue = ?"); + singleEntrySQL.add(srcPath); + + int rows = new SqlExecutor(OntologyManager.getExpSchema()).execute(singleEntrySQL); + LOG.info("Updated " + rows + " row in exp.ObjectProperty for move from " + srcFile + " to " + destFile); + if (rows > 0) + { + // Clear potential object values + OntologyManager.clearPropertyCache(); + } + + // Skip attempting to fix up child paths if we know that the entry is a file. If it's not (either it's a + // directory or it doesn't exist), then try to fix up child records + if (Files.isDirectory(destFile)) + { + if (!srcPath.endsWith(File.separator)) + { + srcPath = srcPath + File.separator; + } + if (!destPath.endsWith(File.separator)) + { + destPath = destPath + File.separator; + } + + // Make the SQL to handle children + SQLFragment childPathsSQL = new SQLFragment(sharedSQL); + childPathsSQL.append(dialect.concatenate(new SQLFragment("?", destPath), new SQLFragment(dialect.getSubstringFunction("StringValue", Integer.toString(srcPath.length() + 1), "5000")))); + childPathsSQL.append(standardWhereSQL); + childPathsSQL.append(" AND "); + childPathsSQL.append(dialect.getStringIndexOfFunction(new SQLFragment("?", srcPath), new SQLFragment("StringValue"))); + childPathsSQL.append(" = 1"); + + int childRows = new SqlExecutor(OntologyManager.getExpSchema()).execute(childPathsSQL); + if (childRows > 0) + { + // Clear potential object values + OntologyManager.clearPropertyCache(); + } + LOG.info("Updated " + childRows + " child paths in exp.ObjectProperty rows for move from " + srcFile + " to " + destFile); + return rows + childRows; + } + return rows; + } + + private int updateHardTables(final Path srcFile, final Path destFile, final User user, final Container container) + { + AtomicInteger result = new AtomicInteger(0); + hardTableFileLinkColumns((schema, table, pathColumn, containerId, domainUri) -> { + // Migrate any paths that match this file move + TableUpdaterFileListener updater = new TableUpdaterFileListener(table, pathColumn.getColumnName(), TableUpdaterFileListener.Type.filePath); + int movedCount = updater.fileMoved(srcFile, destFile, user, container); + result.addAndGet(movedCount); + + if (movedCount > 0 && schema.getName().equalsIgnoreCase(PROVISIONED_SCHEMA_NAME)) + ExpMaterialTableImpl.refreshMaterializedView(domainUri, SampleTypeServiceImpl.SampleChangeType.update); + }); + return result.intValue(); + } + + private interface ForEachFileLinkColumn + { + void exec(DbSchema schema, TableInfo table, ColumnInfo pathColumn, String containerId, @Nullable String domainUri); + } + + private void hardTableFileLinkColumns(final ForEachFileLinkColumn block) + { + // Figure out all of the FileLink columns in hard tables managed by OntologyManager + SQLFragment sql = new SQLFragment("SELECT dd.Container, dd.DomainId, dd.StorageTableName, dd.StorageSchemaName, pd.Name, pd.StorageColumnName FROM "); + sql.append(OntologyManager.getTinfoDomainDescriptor(), "dd"); + sql.append(", "); + sql.append(OntologyManager.getTinfoPropertyDescriptor(), "pd"); + sql.append(", "); + sql.append(OntologyManager.getTinfoPropertyDomain(), "m"); + sql.append(" WHERE dd.DomainId = m.DomainId AND pd.PropertyId = m.PropertyId AND pd.RangeURI = ? "); + sql.add(PropertyType.FILE_LINK.getTypeUri()); + sql.append(" AND dd.StorageTableName IS NOT NULL AND dd.StorageSchemaName IS NOT NULL AND pd.Name IS NOT NULL"); + + new SqlSelector(OntologyManager.getExpSchema(), sql).forEachMap(row -> { + // Find the DbSchema/TableInfo/ColumnInfo for the FileLink column + String storageSchemaName = row.get("StorageSchemaName").toString(); + DbSchema schema = DbSchema.get(storageSchemaName, DbSchemaType.Provisioned); + Domain domain = PropertyService.get().getDomain((Integer) row.get("DomainId")); + // Issue 50781: LKSM: File fields in Sample Types sometimes showing as Text Fields + // Don't use schema.getTable(storageTableName); + if (domain != null) + { + TableInfo tableInfo = StorageProvisioner.get().getSchemaTableInfo(domain); + if (tableInfo != null) + { + String containerId = row.get("Container").toString(); + if (containerId != null) + { + ColumnInfo pathCol = tableInfo.getColumn(row.get("Name").toString()); + // Issue 53502: also try to get tableInfo column by StorageColumnName if not found by Name + if (pathCol == null) + pathCol = tableInfo.getColumn(row.get("StorageColumnName").toString()); + + if (pathCol != null) + block.exec(schema, tableInfo, pathCol, containerId, domain.getTypeURI()); + } + } + } + }); + } + + @Override + public Collection listFiles(@Nullable Container container) + { + Collection files = new ArrayList<>(); + + files.addAll(listObjectPropertyFiles(container)); + files.addAll(listHardTableFiles(container)); + + return files; + } + + private Collection listObjectPropertyFiles(@Nullable Container container) + { + SQLFragment frag = new SQLFragment(); + frag.append("SELECT StringValue\n"); + frag.append("FROM\n"); + frag.append(" ").append(OntologyManager.getTinfoObjectProperty(), "op").append(",\n"); + frag.append(" ").append(OntologyManager.getTinfoObject(), "o").append("\n"); + frag.append("WHERE "); + frag.append(" o.ObjectId = op.ObjectId AND\n"); + if (container != null) + { + frag.append(" o.Container = ? AND \n"); + frag.add(container); + } + frag.append(" PropertyId IN (\n"); + frag.append(" SELECT PropertyId\n"); + frag.append(" FROM ").append(OntologyManager.getTinfoPropertyDescriptor(), "pd").append("\n"); + frag.append(" WHERE RangeURI = ?\n").add(PropertyType.FILE_LINK.getTypeUri()); + frag.append(" )\n"); + + SqlSelector selector = new SqlSelector(OntologyManager.getExpSchema(), frag); + return selector.getArrayList(File.class); + } + + private Collection listHardTableFiles(@NotNull final Container container) + { + final Collection files = new ArrayList<>(); + hardTableFileLinkColumns((schema, table, pathColumn, containerId, domainUri) -> { + TableUpdaterFileListener updater = new TableUpdaterFileListener(table, pathColumn.getColumnName(), TableUpdaterFileListener.Type.filePath); + files.addAll(updater.listFiles(container)); + }); + return files; + } + + @Override + public SQLFragment listFilesQuery() + { + return listFilesQuery(false); + } + + public SQLFragment listFilesQuery(boolean skipCreatedModified) + { + return listFilesQuery(skipCreatedModified, null); + } + + public SQLFragment listFilesQuery(boolean skipCreatedModified, String filePath) + { + final SQLFragment frag = new SQLFragment(); + + // Object property files + frag.append("SELECT\n"); + frag.append(" o.Container,\n"); + if (!skipCreatedModified) + { + frag.append(" NULL AS Created,\n"); + frag.append(" NULL AS CreatedBy,\n"); + frag.append(" NULL AS Modified,\n"); + frag.append(" NULL AS ModifiedBy,\n"); + } + frag.append(" op.StringValue AS FilePath,\n"); + frag.append(" o.ObjectId AS SourceKey,\n"); + frag.append(" 'exp.object' AS SourceName\n"); + frag.append("FROM\n"); + frag.append(" ").append(OntologyManager.getTinfoObjectProperty(), "op").append(",\n"); + frag.append(" ").append(OntologyManager.getTinfoObject(), "o").append("\n"); + frag.append("WHERE\n"); + if (StringUtils.isEmpty(filePath)) + frag.append(" op.StringValue IS NOT NULL AND\n"); + else + frag.append(" op.StringValue = ").appendStringLiteral(filePath, OntologyManager.getTinfoObject().getSqlDialect()).append(" AND\n"); + frag.append(" o.ObjectId = op.ObjectId AND\n"); + frag.append(" PropertyId IN (\n"); + frag.append(" SELECT PropertyId\n"); + frag.append(" FROM ").append(OntologyManager.getTinfoPropertyDescriptor(), "pd").append("\n"); + frag.append(" WHERE RangeURI = ?\n").add(PropertyType.FILE_LINK.getTypeUri()); + frag.append(" )\n"); + + hardTableFileLinkColumns((schema, table, pathColumn, containerId, domainUri) -> { + SQLFragment containerFrag = new SQLFragment("?", containerId); + TableUpdaterFileListener updater = new TableUpdaterFileListener(table, pathColumn.getColumnName(), TableUpdaterFileListener.Type.filePath, null, containerFrag); + frag.append("UNION").append(StringUtils.isEmpty(filePath) ? "" : " ALL" /*keep duplicate*/).append("\n"); + frag.append(updater.listFilesQuery(skipCreatedModified, filePath, false)); + }); + + return frag; + } + + @Override + public SQLFragment listSampleFilesQuery() + { + final SQLFragment frag = new SQLFragment(); + + hardTableFileLinkColumns((schema, table, pathColumn, containerId, domainUri) -> { + if (schema.getName().equals("expsampleset")) + { + SQLFragment containerFrag = new SQLFragment("?", containerId); + TableUpdaterFileListener updater = new TableUpdaterFileListener(table, pathColumn.getColumnName(), TableUpdaterFileListener.Type.filePath, "rowid", containerFrag); + if (!frag.isEmpty()) + frag.append("UNION").append("").append("\n"); + frag.append(updater.listFilesQuery(true, null, true)); + } + }); + + return frag; + } +} diff --git a/filecontent/src/org/labkey/filecontent/FileContentServiceImpl.java b/filecontent/src/org/labkey/filecontent/FileContentServiceImpl.java index 6d732c778a5..b7010a37e6c 100644 --- a/filecontent/src/org/labkey/filecontent/FileContentServiceImpl.java +++ b/filecontent/src/org/labkey/filecontent/FileContentServiceImpl.java @@ -1,1974 +1,1974 @@ -/* - * Copyright (c) 2009-2019 LabKey Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.labkey.filecontent; - -import org.apache.commons.io.FilenameUtils; -import org.apache.commons.lang3.StringUtils; -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; -import org.jetbrains.annotations.NotNull; -import org.jetbrains.annotations.Nullable; -import org.junit.After; -import org.junit.Assert; -import org.junit.Test; -import org.labkey.api.action.SpringActionController; -import org.labkey.api.admin.AdminUrls; -import org.labkey.api.attachments.AttachmentDirectory; -import org.labkey.api.cache.Cache; -import org.labkey.api.cache.CacheManager; -import org.labkey.api.cloud.CloudStoreService; -import org.labkey.api.collections.CaseInsensitiveHashMap; -import org.labkey.api.data.CompareType; -import org.labkey.api.data.Container; -import org.labkey.api.data.ContainerManager; -import org.labkey.api.data.ContainerManager.ContainerListener; -import org.labkey.api.data.ContainerType; -import org.labkey.api.data.CoreSchema; -import org.labkey.api.data.SQLFragment; -import org.labkey.api.data.SimpleFilter; -import org.labkey.api.data.TabContainerType; -import org.labkey.api.data.Table; -import org.labkey.api.data.TableInfo; -import org.labkey.api.data.TableSelector; -import org.labkey.api.data.WorkbookContainerType; -import org.labkey.api.exp.Lsid; -import org.labkey.api.exp.api.ExpData; -import org.labkey.api.exp.api.ExpProtocol; -import org.labkey.api.exp.api.ExpRun; -import org.labkey.api.exp.api.ExperimentService; -import org.labkey.api.exp.query.ExpDataTable; -import org.labkey.api.files.DirectoryPattern; -import org.labkey.api.files.FileContentService; -import org.labkey.api.files.FileListener; -import org.labkey.api.files.FileRoot; -import org.labkey.api.files.FilesAdminOptions; -import org.labkey.api.files.MissingRootDirectoryException; -import org.labkey.api.files.UnsetRootDirectoryException; -import org.labkey.api.files.view.FilesWebPart; -import org.labkey.api.module.Module; -import org.labkey.api.module.ModuleLoader; -import org.labkey.api.pipeline.PipeRoot; -import org.labkey.api.pipeline.PipelineService; -import org.labkey.api.pipeline.PipelineUrls; -import org.labkey.api.query.BatchValidationException; -import org.labkey.api.query.FieldKey; -import org.labkey.api.query.QueryUpdateService; -import org.labkey.api.security.User; -import org.labkey.api.security.permissions.AdminOperationsPermission; -import org.labkey.api.settings.AppProps; -import org.labkey.api.settings.RandomSiteSettingsPropertyHandler; -import org.labkey.api.settings.StartupPropertyEntry; -import org.labkey.api.settings.WriteableAppProps; -import org.labkey.api.test.TestWhen; -import org.labkey.api.util.ContainerUtil; -import org.labkey.api.util.DOM; -import org.labkey.api.util.FileUtil; -import org.labkey.api.util.GUID; -import org.labkey.api.util.HtmlString; -import org.labkey.api.util.NetworkDrive; -import org.labkey.api.util.PageFlowUtil; -import org.labkey.api.util.Path; -import org.labkey.api.util.TestContext; -import org.labkey.api.util.URIUtil; -import org.labkey.api.view.ActionURL; -import org.labkey.api.view.HttpView; -import org.labkey.api.view.ViewBackgroundInfo; -import org.labkey.api.view.ViewContext; -import org.labkey.api.view.template.WarningProvider; -import org.labkey.api.view.template.WarningService; -import org.labkey.api.view.template.Warnings; -import org.labkey.api.webdav.WebdavResource; -import org.labkey.api.webdav.WebdavService; - -import java.beans.PropertyChangeEvent; -import java.io.BufferedWriter; -import java.io.File; -import java.io.IOException; -import java.net.URI; -import java.net.URISyntaxException; -import java.nio.file.Files; -import java.nio.file.InvalidPathException; -import java.nio.file.StandardOpenOption; -import java.util.ArrayList; -import java.util.Collection; -import java.util.Collections; -import java.util.Date; -import java.util.HashMap; -import java.util.HashSet; -import java.util.LinkedHashMap; -import java.util.LinkedHashSet; -import java.util.List; -import java.util.Map; -import java.util.Set; -import java.util.concurrent.CopyOnWriteArrayList; -import java.util.regex.Pattern; -import java.util.stream.Stream; - -import static org.labkey.api.settings.AppProps.SCOPE_SITE_SETTINGS; -import static org.labkey.api.util.DOM.Attribute.href; -import static org.labkey.api.util.DOM.at; - -public class FileContentServiceImpl implements FileContentService, WarningProvider -{ - private static final Logger _log = LogManager.getLogger(FileContentServiceImpl.class); - private static final String UPLOAD_LOG = ".upload.log"; - private static final FileContentServiceImpl INSTANCE = new FileContentServiceImpl(); - - private final ContainerListener _containerListener = new FileContentServiceContainerListener(); - private final List _fileListeners = new CopyOnWriteArrayList<>(); - - private final List _ziploaderPattern = new CopyOnWriteArrayList<>(); - - private volatile boolean _fileRootSetViaStartupProperty = false; - private String _problematicFileRootMessage; - - enum FileAction - { - UPLOAD, - DELETE - } - - static FileContentServiceImpl getInstance() - { - return INSTANCE; - } - - private FileContentServiceImpl() - { - WarningService.get().register(this); - } - - @Override - @NotNull - public List getContainersForFilePath(java.nio.file.Path path) - { - // Ignore cloud files for now - if (FileUtil.hasCloudScheme(path)) - return Collections.emptyList(); - - // If the path is under the default root, do optimistic simple match for containers under the default root - File defaultRoot = getSiteDefaultRoot(); - java.nio.file.Path defaultRootPath = defaultRoot.toPath(); - if (path.startsWith(defaultRootPath)) - { - java.nio.file.Path rel = defaultRootPath.relativize(path); - if (rel.getNameCount() > 0) - { - Container root = ContainerManager.getRoot(); - Container next = root; - while (rel.getNameCount() > 0) - { - // check if there exists a child container that matches the next path segment - java.nio.file.Path top = rel.subpath(0, 1); - assert top != null; - Container child = next.getChild(top.getFileName().toString()); - if (child == null) - break; - - next = child; - - if(rel.getNameCount() > 1) - { - rel = rel.subpath(1, rel.getNameCount()); - } - else - { - break; - } - } - - if (next != null && !next.equals(root)) - { - // verify our naive file path is correct for the container -- it may have a file root other than the default - java.nio.file.Path fileRoot = getFileRootPath(next); - if (fileRoot != null && path.startsWith(fileRoot)) - return Collections.singletonList(next); - } - } - } - - // TODO: Create cache of file root and pipeline root paths -> list of containers - - return Collections.emptyList(); - } - - @Override - public @Nullable File getFileRoot(@NotNull Container c, @NotNull ContentType type) - { - switch (type) - { - case files: - case assayfiles: - String folderName = getFolderName(type); - if (folderName == null) - folderName = ""; - - java.nio.file.Path dir = getFileRootPath(c); - return dir != null ? dir.resolve(folderName).toFile() : null; - - case pipeline: - PipeRoot root = PipelineService.get().findPipelineRoot(c); - return root != null ? root.getRootPath() : null; - } - return null; - } - - @Override - public @Nullable java.nio.file.Path getFileRootPath(@NotNull Container c, @NotNull ContentType type) - { - switch (type) - { - case files: - case assayfiles: - java.nio.file.Path fileRootPath = getFileRootPath(c); - if (null != fileRootPath && !FileUtil.hasCloudScheme(fileRootPath)) // Don't add @files when we're in the cloud - fileRootPath = fileRootPath.resolve(getFolderName(type)); - return fileRootPath; - - case pipeline: - PipeRoot root = PipelineService.get().findPipelineRoot(c); - return root != null ? root.getRootNioPath() : null; - } - return null; - } - - // Returns full uri to file root for this container. filePath is optional relative path to a file under the file root - @Override - public @Nullable URI getFileRootUri(@NotNull Container c, @NotNull ContentType type, @Nullable String filePath) - { - java.nio.file.Path root = FileContentService.get().getFileRootPath(c, FileContentService.ContentType.files); - if (root != null) - { - String path = root.toString(); - if (filePath != null) { - path += filePath; - } - - // non-unix needs a leading slash - if (!path.startsWith("/") && !path.startsWith("\\")) - { - path = "/" + path; - } - return FileUtil.createUri(path); - } - - return null; - } - - @Override - public @Nullable File getFileRoot(@NotNull Container c) - { - java.nio.file.Path path = getFileRootPath(c); - throwIfPathNotFile(path, c); - return path.toFile(); - } - - @Override - public @Nullable java.nio.file.Path getFileRootPath(@NotNull Container c) - { - if (c == null) - return null; - - if (c.isRoot()) - { - return getSiteDefaultRootPath(); - } - - if (!isFileRootDisabled(c)) - { - FileRoot root = FileRootManager.get().getFileRoot(c); - - // check if there is a site wide file root - if (root.getPath() == null || isUseDefaultRoot(c)) - { - return getDefaultRootPath(c, true); - } - else - return getNioPath(c, root.getPath()); - } - return null; - } - - @Override - public File getDefaultRoot(Container c, boolean createDir) - { - return getDefaultRootPath(c, createDir).toFile(); - } - - @Override - public java.nio.file.Path getDefaultRootPath(@NotNull Container c, boolean createDir) - { - Container firstOverride = getFirstAncestorWithOverride(c); - - java.nio.file.Path parentRoot; - if (firstOverride == null) - { - parentRoot = getSiteDefaultRoot().toPath(); - firstOverride = ContainerManager.getRoot(); - } - else - { - parentRoot = getFileRootPath(firstOverride); - } - - if (parentRoot != null && firstOverride != null) - { - java.nio.file.Path fileRootPath; - if (FileUtil.hasCloudScheme(parentRoot)) - { - // For cloud root, we don't have to create directories for this path - fileRootPath = CloudStoreService.get().getPathForOtherContainer(firstOverride, c, FileUtil.pathToString(parentRoot), new Path("")); - } - else - { - // For local, the path may be several directories deep (since it matches the LK folder path), so we should create the directories for that path - fileRootPath = FileUtil.appendPath(parentRoot.toFile(), Path.parse(getRelativePath(c, firstOverride))).toPath(); - - try - { - if (createDir && !NetworkDrive.exists(fileRootPath)) - FileUtil.createDirectories(fileRootPath); - } - catch (IOException e) - { - return null; // throw new RuntimeException(e); TODO: does returning null make certain tests, like TargetedMSQCGuideSetTest pass on Windows? - } - } - - return fileRootPath; - } - return null; - } - - // Return pretty path string for defaultFileRoot and boolean true if defaultFileRoot is cloud - @Override - public DefaultRootInfo getDefaultRootInfo(Container container) - { - String defaultRoot = ""; - boolean isDefaultRootCloud = false; - java.nio.file.Path defaultRootPath = getDefaultRootPath(container, false); - String cloudName = null; - if (defaultRootPath != null) - { - isDefaultRootCloud = FileUtil.hasCloudScheme(defaultRootPath); - if (isDefaultRootCloud && !container.isProject()) - { - FileRoot fileRoot = getDefaultFileRoot(container); - if (null != fileRoot) - defaultRoot = fileRoot.getPath(); - if (null != defaultRoot) - cloudName = getCloudRootName(defaultRoot); - } - else - { - defaultRoot = FileUtil.getAbsolutePath(container, defaultRootPath.toUri()); - } - } - return new DefaultRootInfo(defaultRootPath, defaultRoot, isDefaultRootCloud, cloudName); - } - - @Nullable - // Get FileRoot associated with path returned form getDefaultRootPath() - public FileRoot getDefaultFileRoot(Container c) - { - Container firstOverride = getFirstAncestorWithOverride(c); - - if (firstOverride == null) - firstOverride = ContainerManager.getRoot(); - - if (null != firstOverride) - return FileRootManager.get().getFileRoot(firstOverride); - return null; - } - - private @NotNull String getRelativePath(Container c, Container ancestor) - { - return c.getPath().replaceAll("^" + Pattern.quote(ancestor.getPath()), ""); - } - - //returns the first parent container that has a custom file root, or NULL if none have overrides - private Container getFirstAncestorWithOverride(Container c) - { - Container toTest = c.getParent(); - if (toTest == null) - return null; - - while (isUseDefaultRoot(toTest)) - { - if (toTest == null || toTest.equals(ContainerManager.getRoot())) - return null; - - toTest = toTest.getParent(); - } - - return toTest; - } - - private java.nio.file.Path getNioPath(Container c, @NotNull String fileRootPath) - { - if (isCloudFileRoot(fileRootPath)) - return CloudStoreService.get().getPath(c, getCloudRootName(fileRootPath), new org.labkey.api.util.Path("")); - - return FileUtil.stringToPath(c, fileRootPath, false); // fileRootPath is unencoded - } - - private boolean isCloudFileRoot(String fileRootPseudoPath) - { - return StringUtils.startsWith(fileRootPseudoPath, FileContentService.CLOUD_ROOT_PREFIX); - } - - private String getCloudRootName(@NotNull String fileRootPseudoPath) - { - return fileRootPseudoPath.substring(fileRootPseudoPath.indexOf(FileContentService.CLOUD_ROOT_PREFIX) + FileContentService.CLOUD_ROOT_PREFIX.length() + 1); - } - - @Override - public boolean isCloudRoot(Container c) - { - if (null != c) - { - java.nio.file.Path fileRootPath = getFileRootPath(c); - return null != fileRootPath && FileUtil.hasCloudScheme(fileRootPath); - } - return false; - } - - @Override - @NotNull - public String getCloudRootName(Container c) - { - if (null != c) - { - if (isCloudRoot(c)) - { - FileRoot root = FileRootManager.get().getFileRoot(c); - if (null == root.getPath() || isUseDefaultRoot(c)) - { - Container firstOverride = getFirstAncestorWithOverride(c); - if (null == firstOverride) - firstOverride = ContainerManager.getRoot(); - root = FileRootManager.get().getFileRoot(firstOverride); - if (null == root.getPath()) - return ""; - } - return getCloudRootName(root.getPath()); - } - } - return ""; - } - - @Override - public void setCloudRoot(@NotNull Container c, String cloudRootName) - { - _setFileRoot(c, FileContentService.CLOUD_ROOT_PREFIX + "/" + cloudRootName); - } - - @Override - public void setFileRoot(@NotNull Container c, @Nullable File path) - { - _setFileRoot(c, (null != path ? FileUtil.getAbsoluteCaseSensitiveFile(path).getAbsolutePath() : null)); - } - - @Override - public void setFileRootPath(@NotNull Container c, @Nullable String strPath) - { - String absolutePath = null; - if (strPath != null) - { - URI uri = FileUtil.createUri(strPath, false); // strPath is unencoded - if (FileUtil.hasCloudScheme(uri)) - absolutePath = FileUtil.getAbsolutePath(c, uri); - else - absolutePath = FileUtil.getAbsoluteCaseSensitiveFile(new File(uri)).getAbsolutePath(); - } - _setFileRoot(c, absolutePath); - } - - private void _setFileRoot(@NotNull Container c, @Nullable String absolutePath) - { - if (!c.isContainerFor(ContainerType.DataType.fileRoot)) - throw new IllegalArgumentException("File roots cannot be set for containers of type " + c.getContainerType().getName()); - - FileRoot root = FileRootManager.get().getFileRoot(c); - root.setEnabled(true); - - String oldValue = root.getPath(); - String newValue = null; - - // clear out the root - if (absolutePath == null) - root.setPath(null); - else - { - root.setPath(absolutePath); - newValue = root.getPath(); - } - - FileRootManager.get().saveFileRoot(null, root); - ContainerManager.ContainerPropertyChangeEvent evt = new ContainerManager.ContainerPropertyChangeEvent( - c, ContainerManager.Property.WebRoot, oldValue, newValue); - ContainerManager.firePropertyChangeEvent(evt); - } - - @Override - public void disableFileRoot(Container container) - { - if (container == null || container.isRoot()) - throw new IllegalArgumentException("Disabling either a null project or the root project is not allowed."); - - Container effective = container.getContainerFor(ContainerType.DataType.fileRoot); - if (effective != null) - { - FileRoot root = FileRootManager.get().getFileRoot(effective); - String oldValue = root.getPath(); - root.setEnabled(false); - FileRootManager.get().saveFileRoot(null, root); - - ContainerManager.ContainerPropertyChangeEvent evt = new ContainerManager.ContainerPropertyChangeEvent( - container, ContainerManager.Property.WebRoot, oldValue, null); - ContainerManager.firePropertyChangeEvent(evt); - } - } - - @Override - public boolean isFileRootDisabled(Container c) - { - Container effective = c.getContainerFor(ContainerType.DataType.fileRoot); - if (null == effective) - return false; - - FileRoot root = FileRootManager.get().getFileRoot(effective); - return !root.isEnabled(); - } - - @Override - public boolean isUseDefaultRoot(Container c) - { - if (c == null) - return true; - - Container effective = c.getContainerFor(ContainerType.DataType.fileRoot); - if (null == effective) - return true; - - FileRoot root = FileRootManager.get().getFileRoot(effective); - return root.isUseDefault() || StringUtils.isEmpty(root.getPath()); - } - - @Override - public void setIsUseDefaultRoot(Container c, boolean useDefaultRoot) - { - Container effective = c.getContainerFor(ContainerType.DataType.fileRoot); - if (effective != null) - { - FileRoot root = FileRootManager.get().getFileRoot(effective); - String oldValue = root.getPath(); - root.setEnabled(true); - root.setUseDefault(useDefaultRoot); - if (useDefaultRoot) - root.setPath(null); - FileRootManager.get().saveFileRoot(null, root); - - ContainerManager.ContainerPropertyChangeEvent evt = new ContainerManager.ContainerPropertyChangeEvent( - effective, ContainerManager.Property.WebRoot, oldValue, null); - ContainerManager.firePropertyChangeEvent(evt); - } - } - - @Override - public @NotNull java.nio.file.Path getSiteDefaultRootPath() - { - return getSiteDefaultRoot().toPath(); - } - - @Override - public @NotNull File getSiteDefaultRoot() - { - // Site default is always on file system - File root = AppProps.getInstance().getFileSystemRoot(); - - try - { - if (!NetworkDrive.exists(root)) - { - File configuredRoot = root; - root = getDefaultRoot(); - if (configuredRoot != null && !configuredRoot.equals(root)) - { - String message = "The configured site-wide file root " + configuredRoot + " does not exist. Falling back to " + root; - if (!message.equals(_problematicFileRootMessage)) - { - _problematicFileRootMessage = message; - _log.error(_problematicFileRootMessage); - } - } - } - else - { - _problematicFileRootMessage = null; - } - - if (!NetworkDrive.exists(root)) - { - if (FileUtil.mkdirs(root)) - { - _log.info("Created site-wide file root " + root); - } - else - { - _log.error("Failed when attempting to create site-wide file root " + root); - } - } - } - catch (IOException e) - { - throw new RuntimeException("Unable to create file root directory", e); - } - - return root; - } - - @Override - public String getProblematicFileRootMessage() - { - return _problematicFileRootMessage; - } - - private @NotNull File getDefaultRoot() throws IOException - { - File explodedPath = ModuleLoader.getInstance().getCoreModule().getExplodedPath(); - - File root = explodedPath.getParentFile(); - if (root != null) - { - if (root.getParentFile() != null) - root = root.getParentFile(); - } - File defaultRoot = new File(root, "files"); - if (!NetworkDrive.exists(defaultRoot)) - FileUtil.mkdirs(defaultRoot); - - return defaultRoot; - } - - @Override - public void setSiteDefaultRoot(File root, User user) - { - if (root == null) - throw new IllegalArgumentException("Invalid site root: specified root is null"); - - if (!NetworkDrive.exists(root)) - throw new IllegalArgumentException("Invalid site root: " + root.getAbsolutePath() + " does not exist"); - - File prevRoot = getSiteDefaultRoot(); - WriteableAppProps props = AppProps.getWriteableInstance(); - - props.setFileSystemRoot(root.getAbsolutePath()); - props.save(user); - - FileRootManager.get().clearCache(); - ContainerManager.ContainerPropertyChangeEvent evt = new ContainerManager.ContainerPropertyChangeEvent( - ContainerManager.getRoot(), ContainerManager.Property.SiteRoot, prevRoot, root); - ContainerManager.firePropertyChangeEvent(evt); - } - - @Override - public void setWebfilesEnabled(boolean enabled, User user) - { - WriteableAppProps props = AppProps.getWriteableInstance(); - props.setWebfilesEnabled(enabled); - props.save(user); - } - - @Override - public FileSystemAttachmentParent registerDirectory(Container c, String name, String path, boolean relative) - { - FileSystemAttachmentParent parent = new FileSystemAttachmentParent(); - parent.setContainer(c); - if (null == name) - name = path; - parent.setName(name); - parent.setPath(path); - parent.setRelative(relative); - //We do this because insert does not return new fields - parent.setEntityid(GUID.makeGUID()); - - FileSystemAttachmentParent ret = Table.insert(HttpView.currentContext().getUser(), CoreSchema.getInstance().getMappedDirectories(), parent); - ContainerManager.ContainerPropertyChangeEvent evt = new ContainerManager.ContainerPropertyChangeEvent( - c, ContainerManager.Property.AttachmentDirectory, null, ret); - ContainerManager.firePropertyChangeEvent(evt); - return ret; - } - - @Override - public void unregisterDirectory(Container c, String name) - { - FileSystemAttachmentParent parent = getRegisteredDirectory(c, name); - SimpleFilter filter = SimpleFilter.createContainerFilter(c); - filter.addCondition(FieldKey.fromParts("Name"), name); - Table.delete(CoreSchema.getInstance().getMappedDirectories(), filter); - ContainerManager.ContainerPropertyChangeEvent evt = new ContainerManager.ContainerPropertyChangeEvent( - c, ContainerManager.Property.AttachmentDirectory, parent, null); - ContainerManager.firePropertyChangeEvent(evt); - } - - @Override - public @Nullable AttachmentDirectory getMappedAttachmentDirectory(Container c, boolean createDir) throws UnsetRootDirectoryException, MissingRootDirectoryException - { - return getMappedAttachmentDirectory(c, ContentType.files, createDir); - } - - @Override - @Nullable - public AttachmentDirectory getMappedAttachmentDirectory(Container c, ContentType contentType, boolean createDir) throws UnsetRootDirectoryException - { - try - { - if (createDir) //force create - getMappedDirectory(c, true); - else if (null == getMappedDirectory(c, false)) - return null; - - return new FileSystemAttachmentParent(c, contentType); - } - catch (IOException e) - { - _log.error("Cannot get mapped directory for " + c.getPath(), e); - return null; - } - } - - public java.nio.file.Path getMappedDirectory(Container c, boolean create) throws UnsetRootDirectoryException, IOException - { - java.nio.file.Path root = getFileRootPath(c); - if (!FileUtil.hasCloudScheme(root)) - { - if (null == root) - { - if (create) - throw new UnsetRootDirectoryException(c.isRoot() ? c : c.getProject()); - else - return null; - } - - if (!NetworkDrive.exists(root)) - { - if (create) - throw new MissingRootDirectoryException(c.isRoot() ? c : c.getProject(), root); - else - return null; - - } - } - return root; - } - - @Override - public FileSystemAttachmentParent getRegisteredDirectory(Container c, String name) - { - SimpleFilter filter = SimpleFilter.createContainerFilter(c); - filter.addCondition(FieldKey.fromParts("Name"), name); - - return new TableSelector(CoreSchema.getInstance().getMappedDirectories(), filter, null).getObject(FileSystemAttachmentParent.class); - } - - @Override - public FileSystemAttachmentParent getRegisteredDirectoryFromEntityId(Container c, String entityId) - { - SimpleFilter filter = SimpleFilter.createContainerFilter(c); - filter.addCondition(FieldKey.fromParts("EntityId"), entityId); - - return new TableSelector(CoreSchema.getInstance().getMappedDirectories(), filter, null).getObject(FileSystemAttachmentParent.class); - } - - @Override - public @NotNull Collection getRegisteredDirectories(Container c) - { - SimpleFilter filter = SimpleFilter.createContainerFilter(c); - - return Collections.unmodifiableCollection(new TableSelector(CoreSchema.getInstance().getMappedDirectories(), filter, null).getCollection(FileSystemAttachmentParent.class)); - } - - private class FileContentServiceContainerListener implements ContainerListener - { - @Override - public void containerCreated(Container c, User user) - { - try - { - // Will create directory if it's a default dir - getMappedDirectory(c, false); - } - catch (IOException ex) - { - /* */ - } - } - - @Override - public void containerDeleted(Container c, User user) - { - java.nio.file.Path dir = null; - try - { - // don't delete the file contents if they have a project override - if (isUseDefaultRoot(c) && !isCloudRoot(c)) // Don't do anything for cloud root here. CloudContainerListener will handle - dir = getMappedDirectory(c, false); - - if (null != dir) - { - FileUtil.deleteDir(dir); - } - } - catch (Exception e) - { - _log.error("containerDeleted", e); - } - - ContainerUtil.purgeTable(CoreSchema.getInstance().getMappedDirectories(), c, null); - } - - @Override - public void containerMoved(Container c, Container oldParent, User user) - { - /* **** Cases: - SRC DEST - specific local path same -- no work - specific cloud path same -- no work - local default local default -- move tree - local default cloud default -- move tree - cloud default local default -- move tree - cloud default cloud default -- if change bucket, move tree - *************************************************************/ - if (isUseDefaultRoot(c)) - { - java.nio.file.Path srcParent = getFileRootPath(oldParent); - java.nio.file.Path dest = getFileRootPath(c); - if (null != srcParent && null != dest) - { - if (!FileUtil.hasCloudScheme(srcParent)) - { - File src = new File(srcParent.toFile(), c.getName()); - if (NetworkDrive.exists(src)) - { - if (!FileUtil.hasCloudScheme(dest)) - { - // local -> local - moveFileRoot(src, dest.toFile(), user, c); - } - else - { - // local -> cloud; source starts under @files - File filesSrc = FileUtil.appendName(src, FILES_LINK); - if (NetworkDrive.exists(filesSrc)) - moveFileRoot(filesSrc.toPath(), dest, user, c); - FileUtil.deleteDir(src); // moveFileRoot will delete @files, but we need to delete its parent - } - } - } - else - { - // Get source path using moving container and parent's config (cloudRoot), because that config must be the source config - java.nio.file.Path src = CloudStoreService.get().getPath(c, getCloudRootName(oldParent), new Path("")); - if (!FileUtil.hasCloudScheme(dest)) - { - // cloud -> local; destination is under @files - dest = dest.resolve(FILES_LINK); - moveFileRoot(src, dest, user, c); - } - else - { - // cloud -> cloud - if (!getCloudRootName(oldParent).equals(getCloudRootName(c))) - { - // Different configs - moveFileRoot(src, dest, user, c); - } - } - } - } - } - } - - @Override - public void propertyChange(PropertyChangeEvent propertyChangeEvent) - { - ContainerManager.ContainerPropertyChangeEvent evt = (ContainerManager.ContainerPropertyChangeEvent)propertyChangeEvent; - Container c = evt.container; - - switch (evt.property) - { - case Name: // container rename event - { - String oldValue = (String) propertyChangeEvent.getOldValue(); - String newValue = (String) propertyChangeEvent.getNewValue(); - - java.nio.file.Path location; - try - { - location = getMappedDirectory(c, false); - if (location != null && !FileUtil.hasCloudScheme(location)) // If cloud, folder name for container not dependent on Name - { - //Don't rely on container object. Seems not to point to the - //new location even AFTER rename. Just construct new file paths - File locationFile = location.toFile(); - File parentDir = locationFile.getParentFile(); - File oldLocation = new File(parentDir, oldValue); - File newLocation = new File(parentDir, newValue); - if (NetworkDrive.exists(newLocation)) - moveToDeleted(newLocation); - - if (NetworkDrive.exists(oldLocation)) - { - oldLocation.renameTo(newLocation); - fireFileMoveEvent(oldLocation, newLocation, evt.user, evt.container); - } - } - } - catch (IOException ex) - { - _log.error(ex); - } - - break; - } - } - } - } - - - @Override - public @Nullable String getFolderName(FileContentService.ContentType type) - { - if (type != null) - return "@" + type.name(); - return null; - } - - - /** - * Move the file or directory into a ".deleted" directory under the parent directory. - * @return True if successfully moved. - */ - private static boolean moveToDeleted(File fileToMove) throws IOException - { - if (!NetworkDrive.exists(fileToMove)) - return false; - - File parent = fileToMove.getParentFile(); - - File deletedDir = new File(parent, ".deleted"); - if (!NetworkDrive.exists(deletedDir)) - if (!FileUtil.mkdir(deletedDir)) - return false; - - File newLocation = new File(deletedDir, fileToMove.getName()); - if (NetworkDrive.exists(newLocation)) - FileUtil.deleteDir(newLocation); - - return fileToMove.renameTo(newLocation); - } - - static void logFileAction(java.nio.file.Path directory, String fileName, FileAction action, User user) - { - try (BufferedWriter fw = Files.newBufferedWriter(directory.resolve(UPLOAD_LOG), StandardOpenOption.APPEND, StandardOpenOption.CREATE)) - { - fw.write(action.toString() + "\t" + fileName + "\t" + new Date() + "\t" + (user == null ? "(unknown)" : user.getEmail()) + "\n"); - } - catch (Exception x) - { - //Just log it. - _log.error(x); - } - } - - @Override - public FilesAdminOptions getAdminOptions(Container c) - { - FileRoot root = FileRootManager.get().getFileRoot(c); - String xml = null; - - if (!StringUtils.isBlank(root.getProperties())) - { - xml = root.getProperties(); - } - return new FilesAdminOptions(c, xml); - } - - @Override - public void setAdminOptions(Container c, FilesAdminOptions options) - { - if (options != null) - { - setAdminOptions(c, options.serialize()); - } - } - - @Override - public void setAdminOptions(Container c, String properties) - { - FileRoot root = FileRootManager.get().getFileRoot(c); - - root.setProperties(properties); - FileRootManager.get().saveFileRoot(null, root); - } - - public static final String NAMESPACE_PREFIX = "FileProperties"; - public static final String PROPERTIES_DOMAIN = "File Properties"; - public static final String TYPE_PROPERTIES = "FileProperties"; - - @Override - public String getDomainURI(Container container) - { - return getDomainURI(container, getAdminOptions(container).getFileConfig()); - } - - @Override - public String getDomainURI(Container container, FilesAdminOptions.fileConfig config) - { - while (config == FilesAdminOptions.fileConfig.useParent && container != container.getParent()) - { - container = container.getParent(); - config = getAdminOptions(container).getFileConfig(); - } - - //String typeURI = "urn:lsid:" + AppProps.getInstance().getDefaultLsidAuthority() + ":List" + ".Folder-" + container.getRowId() + ":" + name; - - return new Lsid("urn:lsid:labkey.com:" + NAMESPACE_PREFIX + ".Folder-" + container.getRowId() + ':' + TYPE_PROPERTIES).toString(); - } - - @Override @Nullable - public ExpData getDataObject(WebdavResource resource, Container c) - { - return getDataObject(resource, c, null, false); - } - - @Nullable - private static ExpData getDataObject(WebdavResource resource, Container c, User user, boolean create) - { - // TODO: S3: seems to only be called from Search and currently we're not searching in cloud. SaveCustomPropsAction seems unused - if (resource != null) - { - File file = resource.getFile(); - if (file != null) - { - ExpData data = ExperimentService.get().getExpDataByURL(file, c); - - if (data == null && create) - { - data = ExperimentService.get().createData(c, FileContentService.UPLOADED_FILE); - data.setName(file.getName()); - data.setDataFileURI(file.toURI()); - data.save(user); - } - return data; - } - } - return null; - } - - @Override - public QueryUpdateService getFilePropsUpdateService(TableInfo tinfo, Container container) - { - return new FileQueryUpdateService(tinfo, container); - } - - @Override - public boolean isValidProjectRoot(String root) - { - File f = new File(root); - return NetworkDrive.exists(f) && f.isDirectory(); - } - - @Override - public void moveFileRoot(java.nio.file.Path prev, java.nio.file.Path dest, @Nullable User user, @Nullable Container container) - { - if (!FileUtil.hasCloudScheme(prev) && !FileUtil.hasCloudScheme(dest)) - { - moveFileRoot(prev.toFile(), dest.toFile(), user, container); // Both files; try rename - } - else - { - try - { - // At least one is in the cloud - FileUtil.copyDirectory(prev, dest); - FileUtil.deleteDir(prev); // TODO use more efficient delete - fireFileMoveEvent(prev, dest, user, container); - } - catch (IOException e) - { - _log.error("error occurred moving the file root", e); - } - } - } - - @Override - public void moveFileRoot(File prev, File dest, @Nullable User user, @Nullable Container container) - { - try - { - _log.info("moving " + prev.getPath() + " to " + dest.getPath()); - boolean doRename = true; - - // Our best bet for perf is to do a rename, which doesn't require creating an actual copy. - // If it exists, try deleting the target directory, which will only succeed if it's empty, but would - // enable using renameTo() method. Don't delete if it's a symbolic link, since it wouldn't be recreated - // in the same way. - if (NetworkDrive.exists(dest) && !Files.isSymbolicLink(dest.toPath())) - doRename = dest.delete(); - - if (doRename && !prev.renameTo(dest)) - { - _log.info("rename failed, attempting to copy"); - - //listFiles can return null, which could cause a NPE - File[] children = prev.listFiles(); - if (children != null) - { - for (File file : children) - FileUtil.copyBranch(file, dest); - } - FileUtil.deleteDir(prev); - } - fireFileMoveEvent(prev, dest, user, container); - } - catch (IOException e) - { - _log.error("error occurred moving the file root", e); - } - } - - @Override - public void fireFileCreateEvent(@NotNull File created, @Nullable User user, @Nullable Container container) - { - fireFileCreateEvent(created.toPath(), user, container); - } - - @Override - public void fireFileCreateEvent(@NotNull java.nio.file.Path created, @Nullable User user, @Nullable Container container) - { - java.nio.file.Path absPath = FileUtil.getAbsoluteCaseSensitivePath(container, created); - for (FileListener fileListener : _fileListeners) - { - fileListener.fileCreated(absPath, user, container); - } - } - - @Override - public void fireFileReplacedEvent(@NotNull java.nio.file.Path replaced, @Nullable User user, @Nullable Container container) - { - java.nio.file.Path absPath = FileUtil.getAbsoluteCaseSensitivePath(container, replaced); - for (FileListener fileListener : _fileListeners) - { - fileListener.fileReplaced(absPath, user, container); - } - } - - @Override - public void fireFileDeletedEvent(@NotNull java.nio.file.Path deleted, @Nullable User user, @Nullable Container container) - { - java.nio.file.Path absPath = FileUtil.getAbsoluteCaseSensitivePath(container, deleted); - for (FileListener fileListener : _fileListeners) - { - fileListener.fileDeleted(absPath, user, container); - } - } - - @Override - public int fireFileMoveEvent(@NotNull File src, @NotNull File dest, @Nullable User user, @Nullable Container container) - { - return fireFileMoveEvent(src.toPath(), dest.toPath(), user, container); - } - - @Override - public int fireFileMoveEvent(@NotNull java.nio.file.Path src, @NotNull java.nio.file.Path dest, @Nullable User user, @Nullable Container container) - { - return fireFileMoveEvent(src, dest, user, container, null); - } - - @Override - public int fireFileMoveEvent(@NotNull java.nio.file.Path src, @NotNull java.nio.file.Path dest, @Nullable User user, @Nullable Container sourceContainer, @Nullable Container targetContainer) - { - // Make sure that we've got the best representation of the file that we can - java.nio.file.Path absSrc = FileUtil.getAbsoluteCaseSensitivePath(sourceContainer, src); - java.nio.file.Path absDest = FileUtil.getAbsoluteCaseSensitivePath(targetContainer != null ? targetContainer : sourceContainer, dest); - int result = 0; - for (FileListener fileListener : _fileListeners) - { - result += fileListener.fileMoved(absSrc, absDest, user, sourceContainer, targetContainer); - } - return result; - } - - @Override - public void addFileListener(FileListener listener) - { - _fileListeners.add(listener); - } - - @Override - public Map> listFiles(@NotNull Container container) - { - Map> files = new LinkedHashMap<>(); - for (FileListener fileListener : _fileListeners) - { - files.put(fileListener.getSourceName(), new HashSet<>(fileListener.listFiles(container))); - } - return files; - } - - @Override - public SQLFragment listSampleFilesQuery(@NotNull User currentUser) - { - SQLFragment frag = new SQLFragment(); - String union = ""; - frag.append("("); - - for (FileListener fileListener : _fileListeners) - { - SQLFragment subselect = fileListener.listSampleFilesQuery(); - if (subselect != null) - { - frag.append(union); - frag.append(subselect); - union = "UNION\n"; - } - } - frag.append(")"); - return frag; - } - - @Override - public SQLFragment listFilesQuery(@NotNull User currentUser) - { - SQLFragment frag = new SQLFragment(); - if (currentUser == null || !currentUser.hasSiteAdminPermission()) - { - frag.append("SELECT\n"); - frag.append(" CAST(NULL AS VARCHAR) AS Container,\n"); - frag.append(" NULL AS Created,\n"); - frag.append(" NULL AS CreatedBy,\n"); - frag.append(" NULL AS Modified,\n"); - frag.append(" NULL AS ModifiedBy,\n"); - frag.append(" NULL AS FilePath,\n"); - frag.append(" NULL AS SourceKey,\n"); - frag.append(" NULL AS SourceName\n"); - frag.append("WHERE 1 = 0"); - } - else - { - String union = ""; - frag.append("("); - for (FileListener fileListener : _fileListeners) - { - SQLFragment subselect = fileListener.listFilesQuery(); - if (subselect != null) - { - frag.append(union); - frag.append(subselect); - union = "UNION\n"; - } - } - frag.append(")"); - } - return frag; - } - - @Override - public void setFileRootSetViaStartupProperty(boolean fileRootSetViaStartupProperty) - { - _fileRootSetViaStartupProperty = fileRootSetViaStartupProperty; - } - - @Override - public boolean isFileRootSetViaStartupProperty() - { - return _fileRootSetViaStartupProperty; - } - - public ContainerListener getContainerListener() - { - return _containerListener; - } - - public Set> getNodes(boolean isShowOverridesOnly, @Nullable String browseUrl, Container c) - { - Set> children = new LinkedHashSet<>(); - - try { - java.nio.file.Path assayFilesRoot = getFileRootPath(c, ContentType.assayfiles); - if (NetworkDrive.exists(assayFilesRoot)) - { - Map node = createFileSetNode(c, ASSAY_FILES, assayFilesRoot); - node.put("default", false); - node.put("webdavURL", FilesWebPart.getRootPath(c, ASSAY_FILES).toString()); - children.add(node); - } - - AttachmentDirectory root = getMappedAttachmentDirectory(c, false); - if (root != null) - { - boolean isDefault = isUseDefaultRoot(c); - if (!isDefault || !isShowOverridesOnly) - { - ActionURL config = PageFlowUtil.urlProvider(AdminUrls.class).getProjectSettingsFileURL(c); - Map node = createFileSetNode(c, FILES_LINK, root.getFileSystemDirectoryPath()); - node.put("default", isUseDefaultRoot(c)); - node.put("configureURL", config.getEncodedLocalURIString()); - node.put("browseURL", browseUrl); - node.put("webdavURL", FilesWebPart.getRootPath(c, FILES_LINK).toString()); - - children.add(node); - } - } - - for (AttachmentDirectory fileSet : getRegisteredDirectories(c)) - { - ActionURL config = new ActionURL(FileContentController.ShowAdminAction.class, c); - Map node = createFileSetNode(c, fileSet.getName(), fileSet.getFileSystemDirectoryPath()); - node.put("configureURL", config.getEncodedLocalURIString()); - node.put("browseURL", browseUrl); - node.put("webdavURL", FilesWebPart.getRootPath(c, FILE_SETS_LINK, fileSet.getName()).toString()); - node.put("rootType", "fileset"); - - children.add(node); - } - - PipeRoot pipeRoot = PipelineService.get().findPipelineRoot(c); - if (pipeRoot != null) - { - boolean isDefault = PipelineService.get().hasSiteDefaultRoot(c); - if (!isDefault || !isShowOverridesOnly) - { - ActionURL config = PageFlowUtil.urlProvider(PipelineUrls.class).urlSetup(c); - ActionURL pipelineBrowse = PageFlowUtil.urlProvider(PipelineUrls.class).urlBrowse(c, null); - Map node = createFileSetNode(c, PIPELINE_LINK, pipeRoot.getRootNioPath()); - node.put("default", isDefault ); - node.put("configureURL", config.getEncodedLocalURIString()); - node.put("browseURL", pipelineBrowse.getEncodedLocalURIString()); - node.put("webdavURL", FilesWebPart.getRootPath(c, PIPELINE_LINK).toString()); - - children.add(node); - } - } - } - catch (IOException | UnsetRootDirectoryException ignored) {} - return children; - } - - protected Map createFileSetNode(Container container, String name, java.nio.file.Path dir) - { - Map node = new HashMap<>(); - if (dir != null) - { - node.put("name", name); - node.put("path", FileUtil.getAbsolutePath(container, dir)); - node.put("leaf", true); - } - return node; - } - - public String getAbsolutePathFromDataFileUrl(String dataFileUrl, Container container) - { - return FileUtil.getAbsolutePath(container, FileUtil.createUri(dataFileUrl)); - } - - @Nullable - @Override - public URI getWebDavUrl(@NotNull java.nio.file.Path path, @NotNull Container container, @NotNull PathType type) - { - PipeRoot root = PipelineService.get().getPipelineRootSetting(container); - java.nio.file.Path assayFilesPath = getFileRootPath(container, ContentType.assayfiles); - path = path.toAbsolutePath(); - String relPath = null; - URI rootWebDavUrl = null; - - try - { - // currently, only report if the file is under the parent container - if (root != null && root.isUnderRoot(path)) - { - relPath = root.relativePath(path); - rootWebDavUrl = root.getWebdavURL(); - } - else if (assayFilesPath != null && URIUtil.isDescendant(assayFilesPath.toUri(), path.toUri())) - { - relPath = assayFilesPath.relativize(path).toString(); - rootWebDavUrl = FilesWebPart.getRootPath(container, ASSAY_FILES); - } - - if (relPath != null) - { - relPath = Path.parse(FilenameUtils.separatorsToUnix(relPath)).encode(); - - return switch (type) - { - case folderRelative -> new URI(relPath); - case serverRelative -> new URI(rootWebDavUrl + (rootWebDavUrl.getPath().endsWith("/") ? "" : "/") + relPath); - case full -> new URI(AppProps.getInstance().getBaseServerUrl() + rootWebDavUrl + (rootWebDavUrl.getPath().endsWith("/") ? "" : "/") + relPath); - }; - } - } - catch (InvalidPathException | URISyntaxException e) - { - _log.error("Invalid WebDav URL from: " + path, e); - } - - return null; - } - - @Override - public String getDataFileRelativeFileRootPath(@NotNull String dataFileUrl, Container container) - { - Set> children = getNodes(false, null, container); - String filesRoot = null; // the path for @files - for (Map child : children) - { - String rootName = (String) child.get("name"); - String rootPath = (String) child.get("path"); - - // skip default @pipeline, which is the same as @files - if (PIPELINE_LINK.equals(rootName)) - { - if((boolean) child.get("default") || rootPath.equals(filesRoot)) - continue; - } - - if (FILES_LINK.equals(rootName)) - filesRoot = rootPath; - - String absoluteFilePath = getAbsolutePathFromDataFileUrl(dataFileUrl, container); - if (StringUtils.startsWith(absoluteFilePath, rootPath)) - { - String offset = absoluteFilePath.replace(rootPath, "").replace("\\", "/"); - int lastSlash = offset.lastIndexOf("/"); - if (lastSlash <= 0) - return "/"; - else - return offset.substring(0, lastSlash); - } - } - return null; - } - - @Override - public void ensureFileData(@NotNull ExpDataTable table) - { - Container container = table.getUserSchema().getContainer(); - // The current user may not have insert permission, and they didn't necessarily upload the files anyway - User user = User.getAdminServiceUser(); - QueryUpdateService qus = table.getUpdateService(); - if (qus == null) - { - throw new IllegalArgumentException("getUpdateServer() returned null from " + table); - } - - synchronized (_fileDataUpToDateCache) - { - if (_fileDataUpToDateCache.get(container) != null) // already synced in the past 5 minutes, skip - return; - - _fileDataUpToDateCache.put(container, true); - } - - List existingDataFileUrls = getDataFileUrls(container); - Collection filesets = getRegisteredDirectories(container); - Set> children = getNodes(false, null, container); - String filesRoot = null; // the path for @files - for (Map child : children) - { - String rootName = (String) child.get("name"); - String rootPathVal = (String) child.get("path"); - - // skip default @pipeline, which is the same as @files - if (PIPELINE_LINK.equals(rootName)) - { - if((boolean) child.get("default") || rootPathVal.equals(filesRoot)) - continue; - } - - if (FILES_LINK.equals(rootName)) - filesRoot = rootPathVal; - - String rootDavUrl = (String) child.get("webdavURL"); - - WebdavResource resource = getResource(rootDavUrl); - if (resource == null) - continue; - - List> rows = new ArrayList<>(); - BatchValidationException errors = new BatchValidationException(); - File file = resource.getFile(); - - if (file == null) - { - String rootType = (String) child.get("rootType"); - if ("fileset".equals(rootType)) - { - for (AttachmentDirectory fileset : filesets) - { - if (fileset.getName().equals(rootName)) - { - try - { - file = fileset.getFileSystemDirectory(); - } - catch (MissingRootDirectoryException e) - { - _log.error("Unable to list files for fileset: " + rootName, e); - } - break; - } - } - } - } - - if (file == null) - return; - - try (var ignore = SpringActionController.ignoreSqlUpdates()) - { - java.nio.file.Path rootPath = file.toPath(); - - try (Stream pathStream = Files.walk(rootPath, 100)) // prevent symlink loop - { - pathStream - .filter(path -> !Files.isSymbolicLink(path) && path.compareTo(rootPath) != 0) // exclude symlink & root - .forEach(path -> { - if (!containsUrlOrVariation(existingDataFileUrls, path)) - rows.add(new CaseInsensitiveHashMap<>(Collections.singletonMap("DataFileUrl", path.toUri().toString()))); - }); - } - - qus.insertRows(user, container, rows, errors, null, null); - } - catch (Exception e) - { - _log.error("Error listing content of directory: " + file.getAbsolutePath(), e); - } - } - } - - - @Override - public void addZiploaderPattern(DirectoryPattern directoryPattern) - { - _ziploaderPattern.add(directoryPattern); - } - - @Override - public List getZiploaderPatterns(Container container) - { - List registeredPatterns = new ArrayList<>(); - for(Module module : container.getActiveModules()) - { - _ziploaderPattern.forEach(p -> { - if(p.getModule().getName().equalsIgnoreCase(module.getName())) - registeredPatterns.add(p); - }); - } - return registeredPatterns; - } - - public List getDataFileUrls(Container container) - { - SimpleFilter filter = SimpleFilter.createContainerFilter(container); - filter.addCondition(FieldKey.fromParts("DataFileUrl"), null, CompareType.NONBLANK); - TableSelector selector = new TableSelector(ExperimentService.get().getTinfoData(), Collections.singleton("DataFileUrl"), filter, null); - return selector.getArrayList(String.class); - } - - public Path getPath(String uri) - { - Path path = Path.decode(uri); - - if (!path.startsWith(WebdavService.getPath()) && path.contains(WebdavService.getPath().getName())) - { - String newPath = path.toString(); - int idx = newPath.indexOf(WebdavService.getPath().toString()); - - if (idx != -1) - { - newPath = newPath.substring(idx); - path = Path.parse(newPath); - } - } - return path; - } - - @Nullable - public WebdavResource getResource(String uri) - { - Path path = getPath(uri); - return WebdavService.get().getResolver().lookup(path); - } - - public static void throwIfPathNotFile(java.nio.file.Path path, Container container) - { - if (null == path) - { - throw new RuntimeException("No path to evaluate in " + container.getPath()); - } - if (FileUtil.hasCloudScheme(path)) - { - throw new RuntimeException("Cannot get File object from Cloud File Root in " + container.getPath()); - } - } - - private boolean containsUrlOrVariation(List existingUrls, java.nio.file.Path path) - { - String url = path.toUri().toString(); - if (existingUrls.contains(url)) - return true; - - boolean urlHasTrailingSlash = (Files.isDirectory(path) && (url.endsWith("/") || url.endsWith(File.pathSeparator))); - if (urlHasTrailingSlash && existingUrls.contains(url.substring(0, url.length() - 1))) - return true; - - if (!FileUtil.hasCloudScheme(path)) - { - File file = path.toFile(); - String legacyUrl = file.toURI().toString(); - if (existingUrls.contains(legacyUrl)) // Legacy URI format (file:/users/...) - return true; - - return existingUrls.contains(file.getPath()); - } - return false; - } - - @Override - public File getMoveTargetFile(String absoluteFilePath, @NotNull Container sourceContainer, @NotNull Container targetContainer) - { - if (absoluteFilePath == null) - return null; - - File file = new File(absoluteFilePath); - if (!NetworkDrive.exists(file)) - { - _log.warn("File '" + absoluteFilePath + "' not found and cannot be moved"); - return null; - } - - File sourceFileRoot = getFileRoot(sourceContainer); - if (sourceFileRoot == null) - return null; - - String sourceRootPath = sourceFileRoot.getAbsolutePath(); - if (!absoluteFilePath.startsWith(sourceRootPath)) - { - _log.warn("File '" + absoluteFilePath + "' not currently located in source folder '" + sourceRootPath + "'. Not moving."); - return null; - } - File targetFileRoot = getFileRoot(targetContainer); - if (targetFileRoot == null) - return null; - - String targetPath = absoluteFilePath.replace(sourceRootPath, targetFileRoot.getAbsolutePath()); - File targetFile = new File(targetPath); - return FileUtil.findUniqueFileName(file.getName(), targetFile.getParentFile()); - } - - @Override - public void addDynamicWarnings(@NotNull Warnings warnings, @Nullable ViewContext context, boolean showAllWarnings) - { - if (_problematicFileRootMessage != null && context != null && ContainerManager.getRoot().hasPermission(context.getUser(), AdminOperationsPermission.class)) - { - warnings.add(DOM.createHtmlFragment(_problematicFileRootMessage, " ", DOM.A(at(href, PageFlowUtil.urlProvider(AdminUrls.class).getFilesSiteSettingsURL()), "Configure File System Access"))); - } - else if (showAllWarnings) - { - try - { - warnings.add(HtmlString.of("Configured site-wide file root " + getDefaultRoot() + " does not exist. Falling back to " + getDefaultRoot())); - } - catch (IOException ignored) {} - } - } - - // Cache with short-lived entries so that exp.files can perform reasonably - private static final Cache _fileDataUpToDateCache = CacheManager.getCache(CacheManager.UNLIMITED, 5 * CacheManager.MINUTE, "Files"); - - @TestWhen(TestWhen.When.BVT) - public static class TestCase extends AssertionError - { - private static final String TRICKY_CHARACTERS_FOR_PROJECT_NAMES = "\u2603~!@$&()_+{}-=[],.#\u00E4\u00F6\u00FC"; - - private static final String PROJECT1 = "FileRootTestProject1" + TRICKY_CHARACTERS_FOR_PROJECT_NAMES; - private static final String PROJECT1_SUBFOLDER1 = "Subfolder1"; - private static final String PROJECT1_SUBFOLDER2 = "Subfolder2" + TRICKY_CHARACTERS_FOR_PROJECT_NAMES; - private static final String PROJECT1_SUBSUBFOLDER = "SubSubfolder"; - private static final String PROJECT1_SUBSUBFOLDER_SIBLING = "SubSubfolderSibling"; - private static final String PROJECT2 = "FileRootTestProject2"; - - private static final String FILE_ROOT_SUFFIX = "_FileRootTest"; - private static final String TXT_FILE = "FileContentTestFile.txt"; - - private Map _expectedPaths; - - @Test - public void fileRootsTest() - { - //pre-clean - cleanup(); - - _expectedPaths = new HashMap<>(); - - FileContentService svc = FileContentService.get(); - Assert.assertNotNull(svc); - - Container project1 = ContainerManager.createContainer(ContainerManager.getRoot(), PROJECT1, TestContext.get().getUser()); - _expectedPaths.put(project1, null); - - Container project2 = ContainerManager.createContainer(ContainerManager.getRoot(), PROJECT2, TestContext.get().getUser()); - _expectedPaths.put(project2, null); - - Container subfolder1 = ContainerManager.createContainer(project1, PROJECT1_SUBFOLDER1, TestContext.get().getUser()); - _expectedPaths.put(subfolder1, null); - - Container subfolder2 = ContainerManager.createContainer(project1, PROJECT1_SUBFOLDER2, TestContext.get().getUser()); - _expectedPaths.put(subfolder2, null); - - Container subsubfolder = ContainerManager.createContainer(subfolder1, PROJECT1_SUBSUBFOLDER, TestContext.get().getUser()); - _expectedPaths.put(subsubfolder, null); - - //set custom root on project, then expect children to inherit - File testRoot = getTestRoot(); - - svc.setFileRoot(project1, testRoot); - _expectedPaths.put(project1, testRoot); - - //the subfolder should inherit from the parent - _expectedPaths.put(subfolder1, new File(testRoot, subfolder1.getName())); - assertPathsEqual("Incorrect values returned by getDefaultRoot", _expectedPaths.get(subfolder1), svc.getDefaultRoot(subfolder1, false)); - assertPathsEqual("Subfolder1 has incorrect root", _expectedPaths.get(subfolder1), svc.getFileRoot(subfolder1)); - - _expectedPaths.put(subfolder2, new File(testRoot, subfolder2.getName())); - assertPathsEqual("Incorrect values returned by getDefaultRoot", _expectedPaths.get(subfolder2), svc.getDefaultRoot(subfolder2, false)); - assertPathsEqual("Subfolder2 has incorrect root", _expectedPaths.get(subfolder2), svc.getFileRoot(subfolder2)); - - _expectedPaths.put(subsubfolder, new File(_expectedPaths.get(subfolder1), subsubfolder.getName())); - assertPathsEqual("Incorrect values returned by getDefaultRoot", _expectedPaths.get(subsubfolder), svc.getDefaultRoot(subsubfolder, false)); - assertPathsEqual("SubSubfolder has incorrect root", _expectedPaths.get(subsubfolder), svc.getFileRoot(subsubfolder)); - - //override root on 1st child, expect children of that folder to inherit - _expectedPaths.put(subfolder1, new File(testRoot, "CustomSubfolder")); - _expectedPaths.get(subfolder1).mkdirs(); - svc.setFileRoot(subfolder1, _expectedPaths.get(subfolder1)); - assertPathsEqual("SubSubfolder has incorrect root", new File(_expectedPaths.get(subfolder1), subsubfolder.getName()), svc.getFileRoot(subsubfolder)); - - //reset project, we assume overridden child roots to remain the same - svc.setFileRoot(project1, null); - assertPathsEqual("Subfolder1 has incorrect root", _expectedPaths.get(subfolder1), svc.getFileRoot(subfolder1)); - assertPathsEqual("SubSubfolder has incorrect root", new File(_expectedPaths.get(subfolder1), subsubfolder.getName()), svc.getFileRoot(subsubfolder)); - - } - - private void assertPathsEqual(String msg, File expected, File actual) - { - String expectedPath = FileUtil.getAbsoluteCaseSensitiveFile(expected).getPath(); - String actualPath = FileUtil.getAbsoluteCaseSensitiveFile(actual).getPath(); - Assert.assertEquals(msg, expectedPath, actualPath); - } - - private File getTestRoot() - { - FileContentService svc = FileContentService.get(); - File siteRoot = svc.getSiteDefaultRoot(); - File testRoot = new File(siteRoot, FILE_ROOT_SUFFIX); - testRoot.mkdirs(); - Assert.assertTrue("Unable to create test file root", NetworkDrive.exists(testRoot)); - - return testRoot; - } - - @Test - //when we move a folder, we expect child files to follow, and expect - // any file paths stored in the DB to also get updated - public void testFolderMove() throws Exception - { - //pre-clean - cleanup(); - - _expectedPaths = new HashMap<>(); - - FileContentService svc = FileContentService.get(); - Assert.assertNotNull(svc); - - Container project1 = ContainerManager.createContainer(ContainerManager.getRoot(), PROJECT1, TestContext.get().getUser()); - _expectedPaths.put(project1, null); - - Container project2 = ContainerManager.createContainer(ContainerManager.getRoot(), PROJECT2, TestContext.get().getUser()); - _expectedPaths.put(project2, null); - - Container subfolder1 = ContainerManager.createContainer(project1, PROJECT1_SUBFOLDER1, TestContext.get().getUser()); - _expectedPaths.put(subfolder1, null); - - Container subfolder2 = ContainerManager.createContainer(project1, PROJECT1_SUBFOLDER2, TestContext.get().getUser()); - _expectedPaths.put(subfolder2, null); - - Container subsubfolder = ContainerManager.createContainer(subfolder1, PROJECT1_SUBSUBFOLDER, TestContext.get().getUser()); - Container subsubfolderSibling = ContainerManager.createContainer(subfolder1, PROJECT1_SUBSUBFOLDER_SIBLING, TestContext.get().getUser()); - _expectedPaths.put(subsubfolder, null); - - //create a test file that we will follow - File fileRoot = svc.getFileRoot(subsubfolder, ContentType.files); - fileRoot.mkdirs(); - - File childFile = new File(fileRoot, TXT_FILE); - childFile.createNewFile(); - - ExpData data = ExperimentService.get().createData(subsubfolder, UPLOADED_FILE); - data.setDataFileURI(childFile.toPath().toUri()); - data.save(TestContext.get().getUser()); - - ExpProtocol protocol = ExperimentService.get().createExpProtocol(subsubfolder, ExpProtocol.ApplicationType.ProtocolApplication, "DummyProtocol"); - protocol = ExperimentService.get().insertSimpleProtocol(protocol, TestContext.get().getUser()); - - ExpRun expRun = ExperimentService.get().createExperimentRun(subsubfolder, "DummyRun"); - expRun.setProtocol(protocol); - expRun.setFilePathRootPath(childFile.getParentFile().toPath()); - - ViewBackgroundInfo info = new ViewBackgroundInfo(subsubfolder, TestContext.get().getUser(), null); - ExpRun run = ExperimentService.get().saveSimpleExperimentRun( - expRun, - Collections.emptyMap(), - Collections.singletonMap(data, "Data"), - Collections.emptyMap(), - Collections.emptyMap(), - Collections.emptyMap(), - info, - _log, - false); - - Assert.assertTrue("File not found: " + childFile.getPath(), NetworkDrive.exists(childFile)); - ContainerManager.move(subsubfolder, subfolder2, TestContext.get().getUser()); - Container movedSubfolder = ContainerManager.getChild(subfolder2, subsubfolder.getName()); - - _expectedPaths.put(movedSubfolder, new File(svc.getFileRoot(subfolder2), movedSubfolder.getName())); - assertPathsEqual("Incorrect values returned by getDefaultRoot", _expectedPaths.get(movedSubfolder), svc.getDefaultRoot(movedSubfolder, false)); - assertPathsEqual("SubSubfolder has incorrect root", _expectedPaths.get(movedSubfolder), svc.getFileRoot(movedSubfolder)); - - File expectedFile = new File(svc.getFileRoot(movedSubfolder, ContentType.files), TXT_FILE); - Assert.assertTrue("File was not moved, expected: " + expectedFile.getPath(), NetworkDrive.exists(expectedFile)); - - ExpData movedData = ExperimentService.get().getExpData(data.getRowId()); - Assert.assertNotNull(movedData); - - // Reload the run after it's path has hopefully been updated - expRun = ExperimentService.get().getExpRun(expRun.getRowId()); - - assertPathsEqual("Incorrect data file path", expectedFile, FileUtil.stringToPath(movedSubfolder, movedData.getDataFileUrl()).toFile()); - assertPathsEqual("Incorrect run root path", expectedFile.getParentFile(), expRun.getFilePathRoot()); - - // Issue 38206 - file paths get mangled with multiple folder moves - ContainerManager.move(subsubfolderSibling, subfolder2, TestContext.get().getUser()); - - // Reload the run after it's path has hopefully NOT been updated - expRun = ExperimentService.get().getExpRun(expRun.getRowId()); - assertPathsEqual("Incorrect run root path", expectedFile.getParentFile(), expRun.getFilePathRoot()); - } - - @Test - public void testWorkbooksAndTabs() - { - //pre-clean - cleanup(); - - FileContentService svc = FileContentService.get(); - Assert.assertNotNull(svc); - - Container project1 = ContainerManager.createContainer(ContainerManager.getRoot(), PROJECT1, TestContext.get().getUser()); - - Container workbook = ContainerManager.createContainer(project1, null, null, null, WorkbookContainerType.NAME, TestContext.get().getUser()); - File expectedWorkbookRoot = new File(svc.getFileRoot(project1), workbook.getName()); - assertPathsEqual("Workbook has incorrect file root", expectedWorkbookRoot, svc.getFileRoot(workbook)); - - Container tab = ContainerManager.createContainer(project1, "tab", null, null, TabContainerType.NAME, TestContext.get().getUser()); - File expectedTabRoot = new File(svc.getFileRoot(project1), tab.getName()); - assertPathsEqual("Folder tab has incorrect file root", expectedTabRoot, svc.getFileRoot(tab)); - } - - /** - * Test that the Site Settings can be configured from startup properties - */ - @Test - public void testStartupPropertiesForSiteRootSettings() throws IOException - { - // save the original Site Root File settings so that we can restore them when this test is done - File originalSiteRootFile = FileContentService.get().getSiteDefaultRoot(); - - // create the new site root file to test with as a child of the current site root file so that we know it is in a dir that exist - String originalSiteRootFilePath = originalSiteRootFile.getAbsolutePath(); - File testSiteRootFile = new File(originalSiteRootFilePath, "testSiteRootFile"); - testSiteRootFile.createNewFile(); - - ModuleLoader.getInstance().handleStartupProperties(new RandomSiteSettingsPropertyHandler(){ - @Override - public @NotNull Collection getStartupPropertyEntries() - { - return List.of(new StartupPropertyEntry("siteFileRoot", testSiteRootFile.getAbsolutePath(), "startup", SCOPE_SITE_SETTINGS)); - } - - @Override - public boolean performChecks() - { - return false; - } - }); - - // now check that the expected changes occurred to the Site Root File settings on the server - File newSiteRootFile = FileContentService.get().getSiteDefaultRoot(); - Assert.assertEquals("The expected change in Site Root File was not found", testSiteRootFile.getAbsolutePath(), newSiteRootFile.getAbsolutePath()); - - // restore the Site Root File server settings to how they were originally - FileContentService.get().setSiteDefaultRoot(originalSiteRootFile, null); - testSiteRootFile.delete(); - } - - @After - public void cleanup() - { - FileContentService svc = FileContentService.get(); - Assert.assertNotNull(svc); - - deleteContainerAndFiles(svc, ContainerManager.getForPath(PROJECT1)); - deleteContainerAndFiles(svc, ContainerManager.getForPath(PROJECT2)); - - File testRoot = getTestRoot(); - if (NetworkDrive.exists(testRoot)) - { - FileUtil.deleteDir(testRoot); - } - } - - private void deleteContainerAndFiles(FileContentService svc, @Nullable Container c) - { - if (c != null) - { - ContainerManager.deleteAll(c, TestContext.get().getUser()); - - File file1 = svc.getFileRoot(c); - if (NetworkDrive.exists(file1)) - { - FileUtil.deleteDir(file1); - } - } - } - } -} +/* + * Copyright (c) 2009-2019 LabKey Corporation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.labkey.filecontent; + +import org.apache.commons.io.FilenameUtils; +import org.apache.commons.lang3.StringUtils; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.jetbrains.annotations.NotNull; +import org.jetbrains.annotations.Nullable; +import org.junit.After; +import org.junit.Assert; +import org.junit.Test; +import org.labkey.api.action.SpringActionController; +import org.labkey.api.admin.AdminUrls; +import org.labkey.api.attachments.AttachmentDirectory; +import org.labkey.api.cache.Cache; +import org.labkey.api.cache.CacheManager; +import org.labkey.api.cloud.CloudStoreService; +import org.labkey.api.collections.CaseInsensitiveHashMap; +import org.labkey.api.data.CompareType; +import org.labkey.api.data.Container; +import org.labkey.api.data.ContainerManager; +import org.labkey.api.data.ContainerManager.ContainerListener; +import org.labkey.api.data.ContainerType; +import org.labkey.api.data.CoreSchema; +import org.labkey.api.data.SQLFragment; +import org.labkey.api.data.SimpleFilter; +import org.labkey.api.data.TabContainerType; +import org.labkey.api.data.Table; +import org.labkey.api.data.TableInfo; +import org.labkey.api.data.TableSelector; +import org.labkey.api.data.WorkbookContainerType; +import org.labkey.api.exp.Lsid; +import org.labkey.api.exp.api.ExpData; +import org.labkey.api.exp.api.ExpProtocol; +import org.labkey.api.exp.api.ExpRun; +import org.labkey.api.exp.api.ExperimentService; +import org.labkey.api.exp.query.ExpDataTable; +import org.labkey.api.files.DirectoryPattern; +import org.labkey.api.files.FileContentService; +import org.labkey.api.files.FileListener; +import org.labkey.api.files.FileRoot; +import org.labkey.api.files.FilesAdminOptions; +import org.labkey.api.files.MissingRootDirectoryException; +import org.labkey.api.files.UnsetRootDirectoryException; +import org.labkey.api.files.view.FilesWebPart; +import org.labkey.api.module.Module; +import org.labkey.api.module.ModuleLoader; +import org.labkey.api.pipeline.PipeRoot; +import org.labkey.api.pipeline.PipelineService; +import org.labkey.api.pipeline.PipelineUrls; +import org.labkey.api.query.BatchValidationException; +import org.labkey.api.query.FieldKey; +import org.labkey.api.query.QueryUpdateService; +import org.labkey.api.security.User; +import org.labkey.api.security.permissions.AdminOperationsPermission; +import org.labkey.api.settings.AppProps; +import org.labkey.api.settings.RandomSiteSettingsPropertyHandler; +import org.labkey.api.settings.StartupPropertyEntry; +import org.labkey.api.settings.WriteableAppProps; +import org.labkey.api.test.TestWhen; +import org.labkey.api.util.ContainerUtil; +import org.labkey.api.util.DOM; +import org.labkey.api.util.FileUtil; +import org.labkey.api.util.GUID; +import org.labkey.api.util.HtmlString; +import org.labkey.api.util.NetworkDrive; +import org.labkey.api.util.PageFlowUtil; +import org.labkey.api.util.Path; +import org.labkey.api.util.TestContext; +import org.labkey.api.util.URIUtil; +import org.labkey.api.view.ActionURL; +import org.labkey.api.view.HttpView; +import org.labkey.api.view.ViewBackgroundInfo; +import org.labkey.api.view.ViewContext; +import org.labkey.api.view.template.WarningProvider; +import org.labkey.api.view.template.WarningService; +import org.labkey.api.view.template.Warnings; +import org.labkey.api.webdav.WebdavResource; +import org.labkey.api.webdav.WebdavService; + +import java.beans.PropertyChangeEvent; +import java.io.BufferedWriter; +import java.io.File; +import java.io.IOException; +import java.net.URI; +import java.net.URISyntaxException; +import java.nio.file.Files; +import java.nio.file.InvalidPathException; +import java.nio.file.StandardOpenOption; +import java.util.ArrayList; +import java.util.Collection; +import java.util.Collections; +import java.util.Date; +import java.util.HashMap; +import java.util.HashSet; +import java.util.LinkedHashMap; +import java.util.LinkedHashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.concurrent.CopyOnWriteArrayList; +import java.util.regex.Pattern; +import java.util.stream.Stream; + +import static org.labkey.api.settings.AppProps.SCOPE_SITE_SETTINGS; +import static org.labkey.api.util.DOM.Attribute.href; +import static org.labkey.api.util.DOM.at; + +public class FileContentServiceImpl implements FileContentService, WarningProvider +{ + private static final Logger _log = LogManager.getLogger(FileContentServiceImpl.class); + private static final String UPLOAD_LOG = ".upload.log"; + private static final FileContentServiceImpl INSTANCE = new FileContentServiceImpl(); + + private final ContainerListener _containerListener = new FileContentServiceContainerListener(); + private final List _fileListeners = new CopyOnWriteArrayList<>(); + + private final List _ziploaderPattern = new CopyOnWriteArrayList<>(); + + private volatile boolean _fileRootSetViaStartupProperty = false; + private String _problematicFileRootMessage; + + enum FileAction + { + UPLOAD, + DELETE + } + + static FileContentServiceImpl getInstance() + { + return INSTANCE; + } + + private FileContentServiceImpl() + { + WarningService.get().register(this); + } + + @Override + @NotNull + public List getContainersForFilePath(java.nio.file.Path path) + { + // Ignore cloud files for now + if (FileUtil.hasCloudScheme(path)) + return Collections.emptyList(); + + // If the path is under the default root, do optimistic simple match for containers under the default root + File defaultRoot = getSiteDefaultRoot(); + java.nio.file.Path defaultRootPath = defaultRoot.toPath(); + if (path.startsWith(defaultRootPath)) + { + java.nio.file.Path rel = defaultRootPath.relativize(path); + if (rel.getNameCount() > 0) + { + Container root = ContainerManager.getRoot(); + Container next = root; + while (rel.getNameCount() > 0) + { + // check if there exists a child container that matches the next path segment + java.nio.file.Path top = rel.subpath(0, 1); + assert top != null; + Container child = next.getChild(top.getFileName().toString()); + if (child == null) + break; + + next = child; + + if(rel.getNameCount() > 1) + { + rel = rel.subpath(1, rel.getNameCount()); + } + else + { + break; + } + } + + if (next != null && !next.equals(root)) + { + // verify our naive file path is correct for the container -- it may have a file root other than the default + java.nio.file.Path fileRoot = getFileRootPath(next); + if (fileRoot != null && path.startsWith(fileRoot)) + return Collections.singletonList(next); + } + } + } + + // TODO: Create cache of file root and pipeline root paths -> list of containers + + return Collections.emptyList(); + } + + @Override + public @Nullable File getFileRoot(@NotNull Container c, @NotNull ContentType type) + { + switch (type) + { + case files: + case assayfiles: + String folderName = getFolderName(type); + if (folderName == null) + folderName = ""; + + java.nio.file.Path dir = getFileRootPath(c); + return dir != null ? dir.resolve(folderName).toFile() : null; + + case pipeline: + PipeRoot root = PipelineService.get().findPipelineRoot(c); + return root != null ? root.getRootPath() : null; + } + return null; + } + + @Override + public @Nullable java.nio.file.Path getFileRootPath(@NotNull Container c, @NotNull ContentType type) + { + switch (type) + { + case files: + case assayfiles: + java.nio.file.Path fileRootPath = getFileRootPath(c); + if (null != fileRootPath && !FileUtil.hasCloudScheme(fileRootPath)) // Don't add @files when we're in the cloud + fileRootPath = fileRootPath.resolve(getFolderName(type)); + return fileRootPath; + + case pipeline: + PipeRoot root = PipelineService.get().findPipelineRoot(c); + return root != null ? root.getRootNioPath() : null; + } + return null; + } + + // Returns full uri to file root for this container. filePath is optional relative path to a file under the file root + @Override + public @Nullable URI getFileRootUri(@NotNull Container c, @NotNull ContentType type, @Nullable String filePath) + { + java.nio.file.Path root = FileContentService.get().getFileRootPath(c, FileContentService.ContentType.files); + if (root != null) + { + String path = root.toString(); + if (filePath != null) { + path += filePath; + } + + // non-unix needs a leading slash + if (!path.startsWith("/") && !path.startsWith("\\")) + { + path = "/" + path; + } + return FileUtil.createUri(path); + } + + return null; + } + + @Override + public @Nullable File getFileRoot(@NotNull Container c) + { + java.nio.file.Path path = getFileRootPath(c); + throwIfPathNotFile(path, c); + return path.toFile(); + } + + @Override + public @Nullable java.nio.file.Path getFileRootPath(@NotNull Container c) + { + if (c == null) + return null; + + if (c.isRoot()) + { + return getSiteDefaultRootPath(); + } + + if (!isFileRootDisabled(c)) + { + FileRoot root = FileRootManager.get().getFileRoot(c); + + // check if there is a site wide file root + if (root.getPath() == null || isUseDefaultRoot(c)) + { + return getDefaultRootPath(c, true); + } + else + return getNioPath(c, root.getPath()); + } + return null; + } + + @Override + public File getDefaultRoot(Container c, boolean createDir) + { + return getDefaultRootPath(c, createDir).toFile(); + } + + @Override + public java.nio.file.Path getDefaultRootPath(@NotNull Container c, boolean createDir) + { + Container firstOverride = getFirstAncestorWithOverride(c); + + java.nio.file.Path parentRoot; + if (firstOverride == null) + { + parentRoot = getSiteDefaultRoot().toPath(); + firstOverride = ContainerManager.getRoot(); + } + else + { + parentRoot = getFileRootPath(firstOverride); + } + + if (parentRoot != null && firstOverride != null) + { + java.nio.file.Path fileRootPath; + if (FileUtil.hasCloudScheme(parentRoot)) + { + // For cloud root, we don't have to create directories for this path + fileRootPath = CloudStoreService.get().getPathForOtherContainer(firstOverride, c, FileUtil.pathToString(parentRoot), new Path("")); + } + else + { + // For local, the path may be several directories deep (since it matches the LK folder path), so we should create the directories for that path + fileRootPath = FileUtil.appendPath(parentRoot.toFile(), Path.parse(getRelativePath(c, firstOverride))).toPath(); + + try + { + if (createDir && !NetworkDrive.exists(fileRootPath)) + FileUtil.createDirectories(fileRootPath); + } + catch (IOException e) + { + return null; // throw new RuntimeException(e); TODO: does returning null make certain tests, like TargetedMSQCGuideSetTest pass on Windows? + } + } + + return fileRootPath; + } + return null; + } + + // Return pretty path string for defaultFileRoot and boolean true if defaultFileRoot is cloud + @Override + public DefaultRootInfo getDefaultRootInfo(Container container) + { + String defaultRoot = ""; + boolean isDefaultRootCloud = false; + java.nio.file.Path defaultRootPath = getDefaultRootPath(container, false); + String cloudName = null; + if (defaultRootPath != null) + { + isDefaultRootCloud = FileUtil.hasCloudScheme(defaultRootPath); + if (isDefaultRootCloud && !container.isProject()) + { + FileRoot fileRoot = getDefaultFileRoot(container); + if (null != fileRoot) + defaultRoot = fileRoot.getPath(); + if (null != defaultRoot) + cloudName = getCloudRootName(defaultRoot); + } + else + { + defaultRoot = FileUtil.getAbsolutePath(container, defaultRootPath.toUri()); + } + } + return new DefaultRootInfo(defaultRootPath, defaultRoot, isDefaultRootCloud, cloudName); + } + + @Nullable + // Get FileRoot associated with path returned form getDefaultRootPath() + public FileRoot getDefaultFileRoot(Container c) + { + Container firstOverride = getFirstAncestorWithOverride(c); + + if (firstOverride == null) + firstOverride = ContainerManager.getRoot(); + + if (null != firstOverride) + return FileRootManager.get().getFileRoot(firstOverride); + return null; + } + + private @NotNull String getRelativePath(Container c, Container ancestor) + { + return c.getPath().replaceAll("^" + Pattern.quote(ancestor.getPath()), ""); + } + + //returns the first parent container that has a custom file root, or NULL if none have overrides + private Container getFirstAncestorWithOverride(Container c) + { + Container toTest = c.getParent(); + if (toTest == null) + return null; + + while (isUseDefaultRoot(toTest)) + { + if (toTest == null || toTest.equals(ContainerManager.getRoot())) + return null; + + toTest = toTest.getParent(); + } + + return toTest; + } + + private java.nio.file.Path getNioPath(Container c, @NotNull String fileRootPath) + { + if (isCloudFileRoot(fileRootPath)) + return CloudStoreService.get().getPath(c, getCloudRootName(fileRootPath), new org.labkey.api.util.Path("")); + + return FileUtil.stringToPath(c, fileRootPath, false); // fileRootPath is unencoded + } + + private boolean isCloudFileRoot(String fileRootPseudoPath) + { + return StringUtils.startsWith(fileRootPseudoPath, FileContentService.CLOUD_ROOT_PREFIX); + } + + private String getCloudRootName(@NotNull String fileRootPseudoPath) + { + return fileRootPseudoPath.substring(fileRootPseudoPath.indexOf(FileContentService.CLOUD_ROOT_PREFIX) + FileContentService.CLOUD_ROOT_PREFIX.length() + 1); + } + + @Override + public boolean isCloudRoot(Container c) + { + if (null != c) + { + java.nio.file.Path fileRootPath = getFileRootPath(c); + return null != fileRootPath && FileUtil.hasCloudScheme(fileRootPath); + } + return false; + } + + @Override + @NotNull + public String getCloudRootName(Container c) + { + if (null != c) + { + if (isCloudRoot(c)) + { + FileRoot root = FileRootManager.get().getFileRoot(c); + if (null == root.getPath() || isUseDefaultRoot(c)) + { + Container firstOverride = getFirstAncestorWithOverride(c); + if (null == firstOverride) + firstOverride = ContainerManager.getRoot(); + root = FileRootManager.get().getFileRoot(firstOverride); + if (null == root.getPath()) + return ""; + } + return getCloudRootName(root.getPath()); + } + } + return ""; + } + + @Override + public void setCloudRoot(@NotNull Container c, String cloudRootName) + { + _setFileRoot(c, FileContentService.CLOUD_ROOT_PREFIX + "/" + cloudRootName); + } + + @Override + public void setFileRoot(@NotNull Container c, @Nullable File path) + { + _setFileRoot(c, (null != path ? FileUtil.getAbsoluteCaseSensitiveFile(path).getAbsolutePath() : null)); + } + + @Override + public void setFileRootPath(@NotNull Container c, @Nullable String strPath) + { + String absolutePath = null; + if (strPath != null) + { + URI uri = FileUtil.createUri(strPath, false); // strPath is unencoded + if (FileUtil.hasCloudScheme(uri)) + absolutePath = FileUtil.getAbsolutePath(c, uri); + else + absolutePath = FileUtil.getAbsoluteCaseSensitiveFile(new File(uri)).getAbsolutePath(); + } + _setFileRoot(c, absolutePath); + } + + private void _setFileRoot(@NotNull Container c, @Nullable String absolutePath) + { + if (!c.isContainerFor(ContainerType.DataType.fileRoot)) + throw new IllegalArgumentException("File roots cannot be set for containers of type " + c.getContainerType().getName()); + + FileRoot root = FileRootManager.get().getFileRoot(c); + root.setEnabled(true); + + String oldValue = root.getPath(); + String newValue = null; + + // clear out the root + if (absolutePath == null) + root.setPath(null); + else + { + root.setPath(absolutePath); + newValue = root.getPath(); + } + + FileRootManager.get().saveFileRoot(null, root); + ContainerManager.ContainerPropertyChangeEvent evt = new ContainerManager.ContainerPropertyChangeEvent( + c, ContainerManager.Property.WebRoot, oldValue, newValue); + ContainerManager.firePropertyChangeEvent(evt); + } + + @Override + public void disableFileRoot(Container container) + { + if (container == null || container.isRoot()) + throw new IllegalArgumentException("Disabling either a null project or the root project is not allowed."); + + Container effective = container.getContainerFor(ContainerType.DataType.fileRoot); + if (effective != null) + { + FileRoot root = FileRootManager.get().getFileRoot(effective); + String oldValue = root.getPath(); + root.setEnabled(false); + FileRootManager.get().saveFileRoot(null, root); + + ContainerManager.ContainerPropertyChangeEvent evt = new ContainerManager.ContainerPropertyChangeEvent( + container, ContainerManager.Property.WebRoot, oldValue, null); + ContainerManager.firePropertyChangeEvent(evt); + } + } + + @Override + public boolean isFileRootDisabled(Container c) + { + Container effective = c.getContainerFor(ContainerType.DataType.fileRoot); + if (null == effective) + return false; + + FileRoot root = FileRootManager.get().getFileRoot(effective); + return !root.isEnabled(); + } + + @Override + public boolean isUseDefaultRoot(Container c) + { + if (c == null) + return true; + + Container effective = c.getContainerFor(ContainerType.DataType.fileRoot); + if (null == effective) + return true; + + FileRoot root = FileRootManager.get().getFileRoot(effective); + return root.isUseDefault() || StringUtils.isEmpty(root.getPath()); + } + + @Override + public void setIsUseDefaultRoot(Container c, boolean useDefaultRoot) + { + Container effective = c.getContainerFor(ContainerType.DataType.fileRoot); + if (effective != null) + { + FileRoot root = FileRootManager.get().getFileRoot(effective); + String oldValue = root.getPath(); + root.setEnabled(true); + root.setUseDefault(useDefaultRoot); + if (useDefaultRoot) + root.setPath(null); + FileRootManager.get().saveFileRoot(null, root); + + ContainerManager.ContainerPropertyChangeEvent evt = new ContainerManager.ContainerPropertyChangeEvent( + effective, ContainerManager.Property.WebRoot, oldValue, null); + ContainerManager.firePropertyChangeEvent(evt); + } + } + + @Override + public @NotNull java.nio.file.Path getSiteDefaultRootPath() + { + return getSiteDefaultRoot().toPath(); + } + + @Override + public @NotNull File getSiteDefaultRoot() + { + // Site default is always on file system + File root = AppProps.getInstance().getFileSystemRoot(); + + try + { + if (!NetworkDrive.exists(root)) + { + File configuredRoot = root; + root = getDefaultRoot(); + if (configuredRoot != null && !configuredRoot.equals(root)) + { + String message = "The configured site-wide file root " + configuredRoot + " does not exist. Falling back to " + root; + if (!message.equals(_problematicFileRootMessage)) + { + _problematicFileRootMessage = message; + _log.error(_problematicFileRootMessage); + } + } + } + else + { + _problematicFileRootMessage = null; + } + + if (!NetworkDrive.exists(root)) + { + if (FileUtil.mkdirs(root)) + { + _log.info("Created site-wide file root " + root); + } + else + { + _log.error("Failed when attempting to create site-wide file root " + root); + } + } + } + catch (IOException e) + { + throw new RuntimeException("Unable to create file root directory", e); + } + + return root; + } + + @Override + public String getProblematicFileRootMessage() + { + return _problematicFileRootMessage; + } + + private @NotNull File getDefaultRoot() throws IOException + { + File explodedPath = ModuleLoader.getInstance().getCoreModule().getExplodedPath(); + + File root = explodedPath.getParentFile(); + if (root != null) + { + if (root.getParentFile() != null) + root = root.getParentFile(); + } + File defaultRoot = new File(root, "files"); + if (!NetworkDrive.exists(defaultRoot)) + FileUtil.mkdirs(defaultRoot); + + return defaultRoot; + } + + @Override + public void setSiteDefaultRoot(File root, User user) + { + if (root == null) + throw new IllegalArgumentException("Invalid site root: specified root is null"); + + if (!NetworkDrive.exists(root)) + throw new IllegalArgumentException("Invalid site root: " + root.getAbsolutePath() + " does not exist"); + + File prevRoot = getSiteDefaultRoot(); + WriteableAppProps props = AppProps.getWriteableInstance(); + + props.setFileSystemRoot(root.getAbsolutePath()); + props.save(user); + + FileRootManager.get().clearCache(); + ContainerManager.ContainerPropertyChangeEvent evt = new ContainerManager.ContainerPropertyChangeEvent( + ContainerManager.getRoot(), ContainerManager.Property.SiteRoot, prevRoot, root); + ContainerManager.firePropertyChangeEvent(evt); + } + + @Override + public void setWebfilesEnabled(boolean enabled, User user) + { + WriteableAppProps props = AppProps.getWriteableInstance(); + props.setWebfilesEnabled(enabled); + props.save(user); + } + + @Override + public FileSystemAttachmentParent registerDirectory(Container c, String name, String path, boolean relative) + { + FileSystemAttachmentParent parent = new FileSystemAttachmentParent(); + parent.setContainer(c); + if (null == name) + name = path; + parent.setName(name); + parent.setPath(path); + parent.setRelative(relative); + //We do this because insert does not return new fields + parent.setEntityid(GUID.makeGUID()); + + FileSystemAttachmentParent ret = Table.insert(HttpView.currentContext().getUser(), CoreSchema.getInstance().getMappedDirectories(), parent); + ContainerManager.ContainerPropertyChangeEvent evt = new ContainerManager.ContainerPropertyChangeEvent( + c, ContainerManager.Property.AttachmentDirectory, null, ret); + ContainerManager.firePropertyChangeEvent(evt); + return ret; + } + + @Override + public void unregisterDirectory(Container c, String name) + { + FileSystemAttachmentParent parent = getRegisteredDirectory(c, name); + SimpleFilter filter = SimpleFilter.createContainerFilter(c); + filter.addCondition(FieldKey.fromParts("Name"), name); + Table.delete(CoreSchema.getInstance().getMappedDirectories(), filter); + ContainerManager.ContainerPropertyChangeEvent evt = new ContainerManager.ContainerPropertyChangeEvent( + c, ContainerManager.Property.AttachmentDirectory, parent, null); + ContainerManager.firePropertyChangeEvent(evt); + } + + @Override + public @Nullable AttachmentDirectory getMappedAttachmentDirectory(Container c, boolean createDir) throws UnsetRootDirectoryException, MissingRootDirectoryException + { + return getMappedAttachmentDirectory(c, ContentType.files, createDir); + } + + @Override + @Nullable + public AttachmentDirectory getMappedAttachmentDirectory(Container c, ContentType contentType, boolean createDir) throws UnsetRootDirectoryException + { + try + { + if (createDir) //force create + getMappedDirectory(c, true); + else if (null == getMappedDirectory(c, false)) + return null; + + return new FileSystemAttachmentParent(c, contentType); + } + catch (IOException e) + { + _log.error("Cannot get mapped directory for " + c.getPath(), e); + return null; + } + } + + public java.nio.file.Path getMappedDirectory(Container c, boolean create) throws UnsetRootDirectoryException, IOException + { + java.nio.file.Path root = getFileRootPath(c); + if (!FileUtil.hasCloudScheme(root)) + { + if (null == root) + { + if (create) + throw new UnsetRootDirectoryException(c.isRoot() ? c : c.getProject()); + else + return null; + } + + if (!NetworkDrive.exists(root)) + { + if (create) + throw new MissingRootDirectoryException(c.isRoot() ? c : c.getProject(), root); + else + return null; + + } + } + return root; + } + + @Override + public FileSystemAttachmentParent getRegisteredDirectory(Container c, String name) + { + SimpleFilter filter = SimpleFilter.createContainerFilter(c); + filter.addCondition(FieldKey.fromParts("Name"), name); + + return new TableSelector(CoreSchema.getInstance().getMappedDirectories(), filter, null).getObject(FileSystemAttachmentParent.class); + } + + @Override + public FileSystemAttachmentParent getRegisteredDirectoryFromEntityId(Container c, String entityId) + { + SimpleFilter filter = SimpleFilter.createContainerFilter(c); + filter.addCondition(FieldKey.fromParts("EntityId"), entityId); + + return new TableSelector(CoreSchema.getInstance().getMappedDirectories(), filter, null).getObject(FileSystemAttachmentParent.class); + } + + @Override + public @NotNull Collection getRegisteredDirectories(Container c) + { + SimpleFilter filter = SimpleFilter.createContainerFilter(c); + + return Collections.unmodifiableCollection(new TableSelector(CoreSchema.getInstance().getMappedDirectories(), filter, null).getCollection(FileSystemAttachmentParent.class)); + } + + private class FileContentServiceContainerListener implements ContainerListener + { + @Override + public void containerCreated(Container c, User user) + { + try + { + // Will create directory if it's a default dir + getMappedDirectory(c, false); + } + catch (IOException ex) + { + /* */ + } + } + + @Override + public void containerDeleted(Container c, User user) + { + java.nio.file.Path dir = null; + try + { + // don't delete the file contents if they have a project override + if (isUseDefaultRoot(c) && !isCloudRoot(c)) // Don't do anything for cloud root here. CloudContainerListener will handle + dir = getMappedDirectory(c, false); + + if (null != dir) + { + FileUtil.deleteDir(dir); + } + } + catch (Exception e) + { + _log.error("containerDeleted", e); + } + + ContainerUtil.purgeTable(CoreSchema.getInstance().getMappedDirectories(), c, null); + } + + @Override + public void containerMoved(Container c, Container oldParent, User user) + { + /* **** Cases: + SRC DEST + specific local path same -- no work + specific cloud path same -- no work + local default local default -- move tree + local default cloud default -- move tree + cloud default local default -- move tree + cloud default cloud default -- if change bucket, move tree + *************************************************************/ + if (isUseDefaultRoot(c)) + { + java.nio.file.Path srcParent = getFileRootPath(oldParent); + java.nio.file.Path dest = getFileRootPath(c); + if (null != srcParent && null != dest) + { + if (!FileUtil.hasCloudScheme(srcParent)) + { + File src = new File(srcParent.toFile(), c.getName()); + if (NetworkDrive.exists(src)) + { + if (!FileUtil.hasCloudScheme(dest)) + { + // local -> local + moveFileRoot(src, dest.toFile(), user, c); + } + else + { + // local -> cloud; source starts under @files + File filesSrc = FileUtil.appendName(src, FILES_LINK); + if (NetworkDrive.exists(filesSrc)) + moveFileRoot(filesSrc.toPath(), dest, user, c); + FileUtil.deleteDir(src); // moveFileRoot will delete @files, but we need to delete its parent + } + } + } + else + { + // Get source path using moving container and parent's config (cloudRoot), because that config must be the source config + java.nio.file.Path src = CloudStoreService.get().getPath(c, getCloudRootName(oldParent), new Path("")); + if (!FileUtil.hasCloudScheme(dest)) + { + // cloud -> local; destination is under @files + dest = dest.resolve(FILES_LINK); + moveFileRoot(src, dest, user, c); + } + else + { + // cloud -> cloud + if (!getCloudRootName(oldParent).equals(getCloudRootName(c))) + { + // Different configs + moveFileRoot(src, dest, user, c); + } + } + } + } + } + } + + @Override + public void propertyChange(PropertyChangeEvent propertyChangeEvent) + { + ContainerManager.ContainerPropertyChangeEvent evt = (ContainerManager.ContainerPropertyChangeEvent)propertyChangeEvent; + Container c = evt.container; + + switch (evt.property) + { + case Name: // container rename event + { + String oldValue = (String) propertyChangeEvent.getOldValue(); + String newValue = (String) propertyChangeEvent.getNewValue(); + + java.nio.file.Path location; + try + { + location = getMappedDirectory(c, false); + if (location != null && !FileUtil.hasCloudScheme(location)) // If cloud, folder name for container not dependent on Name + { + //Don't rely on container object. Seems not to point to the + //new location even AFTER rename. Just construct new file paths + File locationFile = location.toFile(); + File parentDir = locationFile.getParentFile(); + File oldLocation = new File(parentDir, oldValue); + File newLocation = new File(parentDir, newValue); + if (NetworkDrive.exists(newLocation)) + moveToDeleted(newLocation); + + if (NetworkDrive.exists(oldLocation)) + { + oldLocation.renameTo(newLocation); + fireFileMoveEvent(oldLocation, newLocation, evt.user, evt.container); + } + } + } + catch (IOException ex) + { + _log.error(ex); + } + + break; + } + } + } + } + + + @Override + public @Nullable String getFolderName(FileContentService.ContentType type) + { + if (type != null) + return "@" + type.name(); + return null; + } + + + /** + * Move the file or directory into a ".deleted" directory under the parent directory. + * @return True if successfully moved. + */ + private static boolean moveToDeleted(File fileToMove) throws IOException + { + if (!NetworkDrive.exists(fileToMove)) + return false; + + File parent = fileToMove.getParentFile(); + + File deletedDir = new File(parent, ".deleted"); + if (!NetworkDrive.exists(deletedDir)) + if (!FileUtil.mkdir(deletedDir)) + return false; + + File newLocation = new File(deletedDir, fileToMove.getName()); + if (NetworkDrive.exists(newLocation)) + FileUtil.deleteDir(newLocation); + + return fileToMove.renameTo(newLocation); + } + + static void logFileAction(java.nio.file.Path directory, String fileName, FileAction action, User user) + { + try (BufferedWriter fw = Files.newBufferedWriter(directory.resolve(UPLOAD_LOG), StandardOpenOption.APPEND, StandardOpenOption.CREATE)) + { + fw.write(action.toString() + "\t" + fileName + "\t" + new Date() + "\t" + (user == null ? "(unknown)" : user.getEmail()) + "\n"); + } + catch (Exception x) + { + //Just log it. + _log.error(x); + } + } + + @Override + public FilesAdminOptions getAdminOptions(Container c) + { + FileRoot root = FileRootManager.get().getFileRoot(c); + String xml = null; + + if (!StringUtils.isBlank(root.getProperties())) + { + xml = root.getProperties(); + } + return new FilesAdminOptions(c, xml); + } + + @Override + public void setAdminOptions(Container c, FilesAdminOptions options) + { + if (options != null) + { + setAdminOptions(c, options.serialize()); + } + } + + @Override + public void setAdminOptions(Container c, String properties) + { + FileRoot root = FileRootManager.get().getFileRoot(c); + + root.setProperties(properties); + FileRootManager.get().saveFileRoot(null, root); + } + + public static final String NAMESPACE_PREFIX = "FileProperties"; + public static final String PROPERTIES_DOMAIN = "File Properties"; + public static final String TYPE_PROPERTIES = "FileProperties"; + + @Override + public String getDomainURI(Container container) + { + return getDomainURI(container, getAdminOptions(container).getFileConfig()); + } + + @Override + public String getDomainURI(Container container, FilesAdminOptions.fileConfig config) + { + while (config == FilesAdminOptions.fileConfig.useParent && container != container.getParent()) + { + container = container.getParent(); + config = getAdminOptions(container).getFileConfig(); + } + + //String typeURI = "urn:lsid:" + AppProps.getInstance().getDefaultLsidAuthority() + ":List" + ".Folder-" + container.getRowId() + ":" + name; + + return new Lsid("urn:lsid:labkey.com:" + NAMESPACE_PREFIX + ".Folder-" + container.getRowId() + ':' + TYPE_PROPERTIES).toString(); + } + + @Override @Nullable + public ExpData getDataObject(WebdavResource resource, Container c) + { + return getDataObject(resource, c, null, false); + } + + @Nullable + private static ExpData getDataObject(WebdavResource resource, Container c, User user, boolean create) + { + // TODO: S3: seems to only be called from Search and currently we're not searching in cloud. SaveCustomPropsAction seems unused + if (resource != null) + { + File file = resource.getFile(); + if (file != null) + { + ExpData data = ExperimentService.get().getExpDataByURL(file, c); + + if (data == null && create) + { + data = ExperimentService.get().createData(c, FileContentService.UPLOADED_FILE); + data.setName(file.getName()); + data.setDataFileURI(file.toURI()); + data.save(user); + } + return data; + } + } + return null; + } + + @Override + public QueryUpdateService getFilePropsUpdateService(TableInfo tinfo, Container container) + { + return new FileQueryUpdateService(tinfo, container); + } + + @Override + public boolean isValidProjectRoot(String root) + { + File f = new File(root); + return NetworkDrive.exists(f) && f.isDirectory(); + } + + @Override + public void moveFileRoot(java.nio.file.Path prev, java.nio.file.Path dest, @Nullable User user, @Nullable Container container) + { + if (!FileUtil.hasCloudScheme(prev) && !FileUtil.hasCloudScheme(dest)) + { + moveFileRoot(prev.toFile(), dest.toFile(), user, container); // Both files; try rename + } + else + { + try + { + // At least one is in the cloud + FileUtil.copyDirectory(prev, dest); + FileUtil.deleteDir(prev); // TODO use more efficient delete + fireFileMoveEvent(prev, dest, user, container); + } + catch (IOException e) + { + _log.error("error occurred moving the file root", e); + } + } + } + + @Override + public void moveFileRoot(File prev, File dest, @Nullable User user, @Nullable Container container) + { + try + { + _log.info("moving " + prev.getPath() + " to " + dest.getPath()); + boolean doRename = true; + + // Our best bet for perf is to do a rename, which doesn't require creating an actual copy. + // If it exists, try deleting the target directory, which will only succeed if it's empty, but would + // enable using renameTo() method. Don't delete if it's a symbolic link, since it wouldn't be recreated + // in the same way. + if (NetworkDrive.exists(dest) && !Files.isSymbolicLink(dest.toPath())) + doRename = dest.delete(); + + if (doRename && !prev.renameTo(dest)) + { + _log.info("rename failed, attempting to copy"); + + //listFiles can return null, which could cause a NPE + File[] children = prev.listFiles(); + if (children != null) + { + for (File file : children) + FileUtil.copyBranch(file, dest); + } + FileUtil.deleteDir(prev); + } + fireFileMoveEvent(prev, dest, user, container); + } + catch (IOException e) + { + _log.error("error occurred moving the file root", e); + } + } + + @Override + public void fireFileCreateEvent(@NotNull File created, @Nullable User user, @Nullable Container container) + { + fireFileCreateEvent(created.toPath(), user, container); + } + + @Override + public void fireFileCreateEvent(@NotNull java.nio.file.Path created, @Nullable User user, @Nullable Container container) + { + java.nio.file.Path absPath = FileUtil.getAbsoluteCaseSensitivePath(container, created); + for (FileListener fileListener : _fileListeners) + { + fileListener.fileCreated(absPath, user, container); + } + } + + @Override + public void fireFileReplacedEvent(@NotNull java.nio.file.Path replaced, @Nullable User user, @Nullable Container container) + { + java.nio.file.Path absPath = FileUtil.getAbsoluteCaseSensitivePath(container, replaced); + for (FileListener fileListener : _fileListeners) + { + fileListener.fileReplaced(absPath, user, container); + } + } + + @Override + public void fireFileDeletedEvent(@NotNull java.nio.file.Path deleted, @Nullable User user, @Nullable Container container) + { + java.nio.file.Path absPath = FileUtil.getAbsoluteCaseSensitivePath(container, deleted); + for (FileListener fileListener : _fileListeners) + { + fileListener.fileDeleted(absPath, user, container); + } + } + + @Override + public int fireFileMoveEvent(@NotNull File src, @NotNull File dest, @Nullable User user, @Nullable Container container) + { + return fireFileMoveEvent(src.toPath(), dest.toPath(), user, container); + } + + @Override + public int fireFileMoveEvent(@NotNull java.nio.file.Path src, @NotNull java.nio.file.Path dest, @Nullable User user, @Nullable Container container) + { + return fireFileMoveEvent(src, dest, user, container, null); + } + + @Override + public int fireFileMoveEvent(@NotNull java.nio.file.Path src, @NotNull java.nio.file.Path dest, @Nullable User user, @Nullable Container sourceContainer, @Nullable Container targetContainer) + { + // Make sure that we've got the best representation of the file that we can + java.nio.file.Path absSrc = FileUtil.getAbsoluteCaseSensitivePath(sourceContainer, src); + java.nio.file.Path absDest = FileUtil.getAbsoluteCaseSensitivePath(targetContainer != null ? targetContainer : sourceContainer, dest); + int result = 0; + for (FileListener fileListener : _fileListeners) + { + result += fileListener.fileMoved(absSrc, absDest, user, sourceContainer, targetContainer); + } + return result; + } + + @Override + public void addFileListener(FileListener listener) + { + _fileListeners.add(listener); + } + + @Override + public Map> listFiles(@NotNull Container container) + { + Map> files = new LinkedHashMap<>(); + for (FileListener fileListener : _fileListeners) + { + files.put(fileListener.getSourceName(), new HashSet<>(fileListener.listFiles(container))); + } + return files; + } + + @Override + public SQLFragment listSampleFilesQuery(@NotNull User currentUser) + { + SQLFragment frag = new SQLFragment(); + String union = ""; + frag.append("("); + + for (FileListener fileListener : _fileListeners) + { + SQLFragment subselect = fileListener.listSampleFilesQuery(); + if (subselect != null) + { + frag.append(union); + frag.append(subselect); + union = "UNION\n"; + } + } + frag.append(")"); + return frag; + } + + @Override + public SQLFragment listFilesQuery(@NotNull User currentUser) + { + SQLFragment frag = new SQLFragment(); + if (currentUser == null || !currentUser.hasSiteAdminPermission()) + { + frag.append("SELECT\n"); + frag.append(" CAST(NULL AS VARCHAR) AS Container,\n"); + frag.append(" NULL AS Created,\n"); + frag.append(" NULL AS CreatedBy,\n"); + frag.append(" NULL AS Modified,\n"); + frag.append(" NULL AS ModifiedBy,\n"); + frag.append(" NULL AS FilePath,\n"); + frag.append(" NULL AS SourceKey,\n"); + frag.append(" NULL AS SourceName\n"); + frag.append("WHERE 1 = 0"); + } + else + { + String union = ""; + frag.append("("); + for (FileListener fileListener : _fileListeners) + { + SQLFragment subselect = fileListener.listFilesQuery(); + if (subselect != null) + { + frag.append(union); + frag.append(subselect); + union = "UNION\n"; + } + } + frag.append(")"); + } + return frag; + } + + @Override + public void setFileRootSetViaStartupProperty(boolean fileRootSetViaStartupProperty) + { + _fileRootSetViaStartupProperty = fileRootSetViaStartupProperty; + } + + @Override + public boolean isFileRootSetViaStartupProperty() + { + return _fileRootSetViaStartupProperty; + } + + public ContainerListener getContainerListener() + { + return _containerListener; + } + + public Set> getNodes(boolean isShowOverridesOnly, @Nullable String browseUrl, Container c) + { + Set> children = new LinkedHashSet<>(); + + try { + java.nio.file.Path assayFilesRoot = getFileRootPath(c, ContentType.assayfiles); + if (NetworkDrive.exists(assayFilesRoot)) + { + Map node = createFileSetNode(c, ASSAY_FILES, assayFilesRoot); + node.put("default", false); + node.put("webdavURL", FilesWebPart.getRootPath(c, ASSAY_FILES).toString()); + children.add(node); + } + + AttachmentDirectory root = getMappedAttachmentDirectory(c, false); + if (root != null) + { + boolean isDefault = isUseDefaultRoot(c); + if (!isDefault || !isShowOverridesOnly) + { + ActionURL config = PageFlowUtil.urlProvider(AdminUrls.class).getProjectSettingsFileURL(c); + Map node = createFileSetNode(c, FILES_LINK, root.getFileSystemDirectoryPath()); + node.put("default", isUseDefaultRoot(c)); + node.put("configureURL", config.getEncodedLocalURIString()); + node.put("browseURL", browseUrl); + node.put("webdavURL", FilesWebPart.getRootPath(c, FILES_LINK).toString()); + + children.add(node); + } + } + + for (AttachmentDirectory fileSet : getRegisteredDirectories(c)) + { + ActionURL config = new ActionURL(FileContentController.ShowAdminAction.class, c); + Map node = createFileSetNode(c, fileSet.getName(), fileSet.getFileSystemDirectoryPath()); + node.put("configureURL", config.getEncodedLocalURIString()); + node.put("browseURL", browseUrl); + node.put("webdavURL", FilesWebPart.getRootPath(c, FILE_SETS_LINK, fileSet.getName()).toString()); + node.put("rootType", "fileset"); + + children.add(node); + } + + PipeRoot pipeRoot = PipelineService.get().findPipelineRoot(c); + if (pipeRoot != null) + { + boolean isDefault = PipelineService.get().hasSiteDefaultRoot(c); + if (!isDefault || !isShowOverridesOnly) + { + ActionURL config = PageFlowUtil.urlProvider(PipelineUrls.class).urlSetup(c); + ActionURL pipelineBrowse = PageFlowUtil.urlProvider(PipelineUrls.class).urlBrowse(c, null); + Map node = createFileSetNode(c, PIPELINE_LINK, pipeRoot.getRootNioPath()); + node.put("default", isDefault ); + node.put("configureURL", config.getEncodedLocalURIString()); + node.put("browseURL", pipelineBrowse.getEncodedLocalURIString()); + node.put("webdavURL", FilesWebPart.getRootPath(c, PIPELINE_LINK).toString()); + + children.add(node); + } + } + } + catch (IOException | UnsetRootDirectoryException ignored) {} + return children; + } + + protected Map createFileSetNode(Container container, String name, java.nio.file.Path dir) + { + Map node = new HashMap<>(); + if (dir != null) + { + node.put("name", name); + node.put("path", FileUtil.getAbsolutePath(container, dir)); + node.put("leaf", true); + } + return node; + } + + public String getAbsolutePathFromDataFileUrl(String dataFileUrl, Container container) + { + return FileUtil.getAbsolutePath(container, FileUtil.createUri(dataFileUrl)); + } + + @Nullable + @Override + public URI getWebDavUrl(@NotNull java.nio.file.Path path, @NotNull Container container, @NotNull PathType type) + { + PipeRoot root = PipelineService.get().getPipelineRootSetting(container); + java.nio.file.Path assayFilesPath = getFileRootPath(container, ContentType.assayfiles); + path = path.toAbsolutePath(); + String relPath = null; + URI rootWebDavUrl = null; + + try + { + // currently, only report if the file is under the parent container + if (root != null && root.isUnderRoot(path)) + { + relPath = root.relativePath(path); + rootWebDavUrl = root.getWebdavURL(); + } + else if (assayFilesPath != null && URIUtil.isDescendant(assayFilesPath.toUri(), path.toUri())) + { + relPath = assayFilesPath.relativize(path).toString(); + rootWebDavUrl = FilesWebPart.getRootPath(container, ASSAY_FILES); + } + + if (relPath != null) + { + relPath = Path.parse(FilenameUtils.separatorsToUnix(relPath)).encode(); + + return switch (type) + { + case folderRelative -> new URI(relPath); + case serverRelative -> new URI(rootWebDavUrl + (rootWebDavUrl.getPath().endsWith("/") ? "" : "/") + relPath); + case full -> new URI(AppProps.getInstance().getBaseServerUrl() + rootWebDavUrl + (rootWebDavUrl.getPath().endsWith("/") ? "" : "/") + relPath); + }; + } + } + catch (InvalidPathException | URISyntaxException e) + { + _log.error("Invalid WebDav URL from: " + path, e); + } + + return null; + } + + @Override + public String getDataFileRelativeFileRootPath(@NotNull String dataFileUrl, Container container) + { + Set> children = getNodes(false, null, container); + String filesRoot = null; // the path for @files + for (Map child : children) + { + String rootName = (String) child.get("name"); + String rootPath = (String) child.get("path"); + + // skip default @pipeline, which is the same as @files + if (PIPELINE_LINK.equals(rootName)) + { + if((boolean) child.get("default") || rootPath.equals(filesRoot)) + continue; + } + + if (FILES_LINK.equals(rootName)) + filesRoot = rootPath; + + String absoluteFilePath = getAbsolutePathFromDataFileUrl(dataFileUrl, container); + if (StringUtils.startsWith(absoluteFilePath, rootPath)) + { + String offset = absoluteFilePath.replace(rootPath, "").replace("\\", "/"); + int lastSlash = offset.lastIndexOf("/"); + if (lastSlash <= 0) + return "/"; + else + return offset.substring(0, lastSlash); + } + } + return null; + } + + @Override + public void ensureFileData(@NotNull ExpDataTable table) + { + Container container = table.getUserSchema().getContainer(); + // The current user may not have insert permission, and they didn't necessarily upload the files anyway + User user = User.getAdminServiceUser(); + QueryUpdateService qus = table.getUpdateService(); + if (qus == null) + { + throw new IllegalArgumentException("getUpdateServer() returned null from " + table); + } + + synchronized (_fileDataUpToDateCache) + { + if (_fileDataUpToDateCache.get(container) != null) // already synced in the past 5 minutes, skip + return; + + _fileDataUpToDateCache.put(container, true); + } + + List existingDataFileUrls = getDataFileUrls(container); + Collection filesets = getRegisteredDirectories(container); + Set> children = getNodes(false, null, container); + String filesRoot = null; // the path for @files + for (Map child : children) + { + String rootName = (String) child.get("name"); + String rootPathVal = (String) child.get("path"); + + // skip default @pipeline, which is the same as @files + if (PIPELINE_LINK.equals(rootName)) + { + if((boolean) child.get("default") || rootPathVal.equals(filesRoot)) + continue; + } + + if (FILES_LINK.equals(rootName)) + filesRoot = rootPathVal; + + String rootDavUrl = (String) child.get("webdavURL"); + + WebdavResource resource = getResource(rootDavUrl); + if (resource == null) + continue; + + List> rows = new ArrayList<>(); + BatchValidationException errors = new BatchValidationException(); + File file = resource.getFile(); + + if (file == null) + { + String rootType = (String) child.get("rootType"); + if ("fileset".equals(rootType)) + { + for (AttachmentDirectory fileset : filesets) + { + if (fileset.getName().equals(rootName)) + { + try + { + file = fileset.getFileSystemDirectory(); + } + catch (MissingRootDirectoryException e) + { + _log.error("Unable to list files for fileset: " + rootName, e); + } + break; + } + } + } + } + + if (file == null) + return; + + try (var ignore = SpringActionController.ignoreSqlUpdates()) + { + java.nio.file.Path rootPath = file.toPath(); + + try (Stream pathStream = Files.walk(rootPath, 100)) // prevent symlink loop + { + pathStream + .filter(path -> !Files.isSymbolicLink(path) && path.compareTo(rootPath) != 0) // exclude symlink & root + .forEach(path -> { + if (!containsUrlOrVariation(existingDataFileUrls, path)) + rows.add(new CaseInsensitiveHashMap<>(Collections.singletonMap("DataFileUrl", path.toUri().toString()))); + }); + } + + qus.insertRows(user, container, rows, errors, null, null); + } + catch (Exception e) + { + _log.error("Error listing content of directory: " + file.getAbsolutePath(), e); + } + } + } + + + @Override + public void addZiploaderPattern(DirectoryPattern directoryPattern) + { + _ziploaderPattern.add(directoryPattern); + } + + @Override + public List getZiploaderPatterns(Container container) + { + List registeredPatterns = new ArrayList<>(); + for(Module module : container.getActiveModules()) + { + _ziploaderPattern.forEach(p -> { + if(p.getModule().getName().equalsIgnoreCase(module.getName())) + registeredPatterns.add(p); + }); + } + return registeredPatterns; + } + + public List getDataFileUrls(Container container) + { + SimpleFilter filter = SimpleFilter.createContainerFilter(container); + filter.addCondition(FieldKey.fromParts("DataFileUrl"), null, CompareType.NONBLANK); + TableSelector selector = new TableSelector(ExperimentService.get().getTinfoData(), Collections.singleton("DataFileUrl"), filter, null); + return selector.getArrayList(String.class); + } + + public Path getPath(String uri) + { + Path path = Path.decode(uri); + + if (!path.startsWith(WebdavService.getPath()) && path.contains(WebdavService.getPath().getName())) + { + String newPath = path.toString(); + int idx = newPath.indexOf(WebdavService.getPath().toString()); + + if (idx != -1) + { + newPath = newPath.substring(idx); + path = Path.parse(newPath); + } + } + return path; + } + + @Nullable + public WebdavResource getResource(String uri) + { + Path path = getPath(uri); + return WebdavService.get().getResolver().lookup(path); + } + + public static void throwIfPathNotFile(java.nio.file.Path path, Container container) + { + if (null == path) + { + throw new RuntimeException("No path to evaluate in " + container.getPath()); + } + if (FileUtil.hasCloudScheme(path)) + { + throw new RuntimeException("Cannot get File object from Cloud File Root in " + container.getPath()); + } + } + + private boolean containsUrlOrVariation(List existingUrls, java.nio.file.Path path) + { + String url = path.toUri().toString(); + if (existingUrls.contains(url)) + return true; + + boolean urlHasTrailingSlash = (Files.isDirectory(path) && (url.endsWith("/") || url.endsWith(File.pathSeparator))); + if (urlHasTrailingSlash && existingUrls.contains(url.substring(0, url.length() - 1))) + return true; + + if (!FileUtil.hasCloudScheme(path)) + { + File file = path.toFile(); + String legacyUrl = file.toURI().toString(); + if (existingUrls.contains(legacyUrl)) // Legacy URI format (file:/users/...) + return true; + + return existingUrls.contains(file.getPath()); + } + return false; + } + + @Override + public File getMoveTargetFile(String absoluteFilePath, @NotNull Container sourceContainer, @NotNull Container targetContainer) + { + if (absoluteFilePath == null) + return null; + + File file = new File(absoluteFilePath); + if (!NetworkDrive.exists(file)) + { + _log.warn("File '" + absoluteFilePath + "' not found and cannot be moved"); + return null; + } + + File sourceFileRoot = getFileRoot(sourceContainer); + if (sourceFileRoot == null) + return null; + + String sourceRootPath = sourceFileRoot.getAbsolutePath(); + if (!absoluteFilePath.startsWith(sourceRootPath)) + { + _log.warn("File '" + absoluteFilePath + "' not currently located in source folder '" + sourceRootPath + "'. Not moving."); + return null; + } + File targetFileRoot = getFileRoot(targetContainer); + if (targetFileRoot == null) + return null; + + String targetPath = absoluteFilePath.replace(sourceRootPath, targetFileRoot.getAbsolutePath()); + File targetFile = new File(targetPath); + return FileUtil.findUniqueFileName(file.getName(), targetFile.getParentFile()); + } + + @Override + public void addDynamicWarnings(@NotNull Warnings warnings, @Nullable ViewContext context, boolean showAllWarnings) + { + if (_problematicFileRootMessage != null && context != null && ContainerManager.getRoot().hasPermission(context.getUser(), AdminOperationsPermission.class)) + { + warnings.add(DOM.createHtmlFragment(_problematicFileRootMessage, " ", DOM.A(at(href, PageFlowUtil.urlProvider(AdminUrls.class).getFilesSiteSettingsURL()), "Configure File System Access"))); + } + else if (showAllWarnings) + { + try + { + warnings.add(HtmlString.of("Configured site-wide file root " + getDefaultRoot() + " does not exist. Falling back to " + getDefaultRoot())); + } + catch (IOException ignored) {} + } + } + + // Cache with short-lived entries so that exp.files can perform reasonably + private static final Cache _fileDataUpToDateCache = CacheManager.getCache(CacheManager.UNLIMITED, 5 * CacheManager.MINUTE, "Files"); + + @TestWhen(TestWhen.When.BVT) + public static class TestCase extends AssertionError + { + private static final String TRICKY_CHARACTERS_FOR_PROJECT_NAMES = "\u2603~!@$&()_+{}-=[],.#\u00E4\u00F6\u00FC"; + + private static final String PROJECT1 = "FileRootTestProject1" + TRICKY_CHARACTERS_FOR_PROJECT_NAMES; + private static final String PROJECT1_SUBFOLDER1 = "Subfolder1"; + private static final String PROJECT1_SUBFOLDER2 = "Subfolder2" + TRICKY_CHARACTERS_FOR_PROJECT_NAMES; + private static final String PROJECT1_SUBSUBFOLDER = "SubSubfolder"; + private static final String PROJECT1_SUBSUBFOLDER_SIBLING = "SubSubfolderSibling"; + private static final String PROJECT2 = "FileRootTestProject2"; + + private static final String FILE_ROOT_SUFFIX = "_FileRootTest"; + private static final String TXT_FILE = "FileContentTestFile.txt"; + + private Map _expectedPaths; + + @Test + public void fileRootsTest() + { + //pre-clean + cleanup(); + + _expectedPaths = new HashMap<>(); + + FileContentService svc = FileContentService.get(); + Assert.assertNotNull(svc); + + Container project1 = ContainerManager.createContainer(ContainerManager.getRoot(), PROJECT1, TestContext.get().getUser()); + _expectedPaths.put(project1, null); + + Container project2 = ContainerManager.createContainer(ContainerManager.getRoot(), PROJECT2, TestContext.get().getUser()); + _expectedPaths.put(project2, null); + + Container subfolder1 = ContainerManager.createContainer(project1, PROJECT1_SUBFOLDER1, TestContext.get().getUser()); + _expectedPaths.put(subfolder1, null); + + Container subfolder2 = ContainerManager.createContainer(project1, PROJECT1_SUBFOLDER2, TestContext.get().getUser()); + _expectedPaths.put(subfolder2, null); + + Container subsubfolder = ContainerManager.createContainer(subfolder1, PROJECT1_SUBSUBFOLDER, TestContext.get().getUser()); + _expectedPaths.put(subsubfolder, null); + + //set custom root on project, then expect children to inherit + File testRoot = getTestRoot(); + + svc.setFileRoot(project1, testRoot); + _expectedPaths.put(project1, testRoot); + + //the subfolder should inherit from the parent + _expectedPaths.put(subfolder1, new File(testRoot, subfolder1.getName())); + assertPathsEqual("Incorrect values returned by getDefaultRoot", _expectedPaths.get(subfolder1), svc.getDefaultRoot(subfolder1, false)); + assertPathsEqual("Subfolder1 has incorrect root", _expectedPaths.get(subfolder1), svc.getFileRoot(subfolder1)); + + _expectedPaths.put(subfolder2, new File(testRoot, subfolder2.getName())); + assertPathsEqual("Incorrect values returned by getDefaultRoot", _expectedPaths.get(subfolder2), svc.getDefaultRoot(subfolder2, false)); + assertPathsEqual("Subfolder2 has incorrect root", _expectedPaths.get(subfolder2), svc.getFileRoot(subfolder2)); + + _expectedPaths.put(subsubfolder, new File(_expectedPaths.get(subfolder1), subsubfolder.getName())); + assertPathsEqual("Incorrect values returned by getDefaultRoot", _expectedPaths.get(subsubfolder), svc.getDefaultRoot(subsubfolder, false)); + assertPathsEqual("SubSubfolder has incorrect root", _expectedPaths.get(subsubfolder), svc.getFileRoot(subsubfolder)); + + //override root on 1st child, expect children of that folder to inherit + _expectedPaths.put(subfolder1, new File(testRoot, "CustomSubfolder")); + _expectedPaths.get(subfolder1).mkdirs(); + svc.setFileRoot(subfolder1, _expectedPaths.get(subfolder1)); + assertPathsEqual("SubSubfolder has incorrect root", new File(_expectedPaths.get(subfolder1), subsubfolder.getName()), svc.getFileRoot(subsubfolder)); + + //reset project, we assume overridden child roots to remain the same + svc.setFileRoot(project1, null); + assertPathsEqual("Subfolder1 has incorrect root", _expectedPaths.get(subfolder1), svc.getFileRoot(subfolder1)); + assertPathsEqual("SubSubfolder has incorrect root", new File(_expectedPaths.get(subfolder1), subsubfolder.getName()), svc.getFileRoot(subsubfolder)); + + } + + private void assertPathsEqual(String msg, File expected, File actual) + { + String expectedPath = FileUtil.getAbsoluteCaseSensitiveFile(expected).getPath(); + String actualPath = FileUtil.getAbsoluteCaseSensitiveFile(actual).getPath(); + Assert.assertEquals(msg, expectedPath, actualPath); + } + + private File getTestRoot() + { + FileContentService svc = FileContentService.get(); + File siteRoot = svc.getSiteDefaultRoot(); + File testRoot = new File(siteRoot, FILE_ROOT_SUFFIX); + testRoot.mkdirs(); + Assert.assertTrue("Unable to create test file root", NetworkDrive.exists(testRoot)); + + return testRoot; + } + + @Test + //when we move a folder, we expect child files to follow, and expect + // any file paths stored in the DB to also get updated + public void testFolderMove() throws Exception + { + //pre-clean + cleanup(); + + _expectedPaths = new HashMap<>(); + + FileContentService svc = FileContentService.get(); + Assert.assertNotNull(svc); + + Container project1 = ContainerManager.createContainer(ContainerManager.getRoot(), PROJECT1, TestContext.get().getUser()); + _expectedPaths.put(project1, null); + + Container project2 = ContainerManager.createContainer(ContainerManager.getRoot(), PROJECT2, TestContext.get().getUser()); + _expectedPaths.put(project2, null); + + Container subfolder1 = ContainerManager.createContainer(project1, PROJECT1_SUBFOLDER1, TestContext.get().getUser()); + _expectedPaths.put(subfolder1, null); + + Container subfolder2 = ContainerManager.createContainer(project1, PROJECT1_SUBFOLDER2, TestContext.get().getUser()); + _expectedPaths.put(subfolder2, null); + + Container subsubfolder = ContainerManager.createContainer(subfolder1, PROJECT1_SUBSUBFOLDER, TestContext.get().getUser()); + Container subsubfolderSibling = ContainerManager.createContainer(subfolder1, PROJECT1_SUBSUBFOLDER_SIBLING, TestContext.get().getUser()); + _expectedPaths.put(subsubfolder, null); + + //create a test file that we will follow + File fileRoot = svc.getFileRoot(subsubfolder, ContentType.files); + fileRoot.mkdirs(); + + File childFile = new File(fileRoot, TXT_FILE); + childFile.createNewFile(); + + ExpData data = ExperimentService.get().createData(subsubfolder, UPLOADED_FILE); + data.setDataFileURI(childFile.toPath().toUri()); + data.save(TestContext.get().getUser()); + + ExpProtocol protocol = ExperimentService.get().createExpProtocol(subsubfolder, ExpProtocol.ApplicationType.ProtocolApplication, "DummyProtocol"); + protocol = ExperimentService.get().insertSimpleProtocol(protocol, TestContext.get().getUser()); + + ExpRun expRun = ExperimentService.get().createExperimentRun(subsubfolder, "DummyRun"); + expRun.setProtocol(protocol); + expRun.setFilePathRootPath(childFile.getParentFile().toPath()); + + ViewBackgroundInfo info = new ViewBackgroundInfo(subsubfolder, TestContext.get().getUser(), null); + ExpRun run = ExperimentService.get().saveSimpleExperimentRun( + expRun, + Collections.emptyMap(), + Collections.singletonMap(data, "Data"), + Collections.emptyMap(), + Collections.emptyMap(), + Collections.emptyMap(), + info, + _log, + false); + + Assert.assertTrue("File not found: " + childFile.getPath(), NetworkDrive.exists(childFile)); + ContainerManager.move(subsubfolder, subfolder2, TestContext.get().getUser()); + Container movedSubfolder = ContainerManager.getChild(subfolder2, subsubfolder.getName()); + + _expectedPaths.put(movedSubfolder, new File(svc.getFileRoot(subfolder2), movedSubfolder.getName())); + assertPathsEqual("Incorrect values returned by getDefaultRoot", _expectedPaths.get(movedSubfolder), svc.getDefaultRoot(movedSubfolder, false)); + assertPathsEqual("SubSubfolder has incorrect root", _expectedPaths.get(movedSubfolder), svc.getFileRoot(movedSubfolder)); + + File expectedFile = new File(svc.getFileRoot(movedSubfolder, ContentType.files), TXT_FILE); + Assert.assertTrue("File was not moved, expected: " + expectedFile.getPath(), NetworkDrive.exists(expectedFile)); + + ExpData movedData = ExperimentService.get().getExpData(data.getRowId()); + Assert.assertNotNull(movedData); + + // Reload the run after it's path has hopefully been updated + expRun = ExperimentService.get().getExpRun(expRun.getRowId()); + + assertPathsEqual("Incorrect data file path", expectedFile, FileUtil.stringToPath(movedSubfolder, movedData.getDataFileUrl()).toFile()); + assertPathsEqual("Incorrect run root path", expectedFile.getParentFile(), expRun.getFilePathRoot()); + + // Issue 38206 - file paths get mangled with multiple folder moves + ContainerManager.move(subsubfolderSibling, subfolder2, TestContext.get().getUser()); + + // Reload the run after it's path has hopefully NOT been updated + expRun = ExperimentService.get().getExpRun(expRun.getRowId()); + assertPathsEqual("Incorrect run root path", expectedFile.getParentFile(), expRun.getFilePathRoot()); + } + + @Test + public void testWorkbooksAndTabs() + { + //pre-clean + cleanup(); + + FileContentService svc = FileContentService.get(); + Assert.assertNotNull(svc); + + Container project1 = ContainerManager.createContainer(ContainerManager.getRoot(), PROJECT1, TestContext.get().getUser()); + + Container workbook = ContainerManager.createContainer(project1, null, null, null, WorkbookContainerType.NAME, TestContext.get().getUser()); + File expectedWorkbookRoot = new File(svc.getFileRoot(project1), workbook.getName()); + assertPathsEqual("Workbook has incorrect file root", expectedWorkbookRoot, svc.getFileRoot(workbook)); + + Container tab = ContainerManager.createContainer(project1, "tab", null, null, TabContainerType.NAME, TestContext.get().getUser()); + File expectedTabRoot = new File(svc.getFileRoot(project1), tab.getName()); + assertPathsEqual("Folder tab has incorrect file root", expectedTabRoot, svc.getFileRoot(tab)); + } + + /** + * Test that the Site Settings can be configured from startup properties + */ + @Test + public void testStartupPropertiesForSiteRootSettings() throws IOException + { + // save the original Site Root File settings so that we can restore them when this test is done + File originalSiteRootFile = FileContentService.get().getSiteDefaultRoot(); + + // create the new site root file to test with as a child of the current site root file so that we know it is in a dir that exist + String originalSiteRootFilePath = originalSiteRootFile.getAbsolutePath(); + File testSiteRootFile = new File(originalSiteRootFilePath, "testSiteRootFile"); + testSiteRootFile.createNewFile(); + + ModuleLoader.getInstance().handleStartupProperties(new RandomSiteSettingsPropertyHandler(){ + @Override + public @NotNull Collection getStartupPropertyEntries() + { + return List.of(new StartupPropertyEntry("siteFileRoot", testSiteRootFile.getAbsolutePath(), "startup", SCOPE_SITE_SETTINGS)); + } + + @Override + public boolean performChecks() + { + return false; + } + }); + + // now check that the expected changes occurred to the Site Root File settings on the server + File newSiteRootFile = FileContentService.get().getSiteDefaultRoot(); + Assert.assertEquals("The expected change in Site Root File was not found", testSiteRootFile.getAbsolutePath(), newSiteRootFile.getAbsolutePath()); + + // restore the Site Root File server settings to how they were originally + FileContentService.get().setSiteDefaultRoot(originalSiteRootFile, null); + testSiteRootFile.delete(); + } + + @After + public void cleanup() + { + FileContentService svc = FileContentService.get(); + Assert.assertNotNull(svc); + + deleteContainerAndFiles(svc, ContainerManager.getForPath(PROJECT1)); + deleteContainerAndFiles(svc, ContainerManager.getForPath(PROJECT2)); + + File testRoot = getTestRoot(); + if (NetworkDrive.exists(testRoot)) + { + FileUtil.deleteDir(testRoot); + } + } + + private void deleteContainerAndFiles(FileContentService svc, @Nullable Container c) + { + if (c != null) + { + ContainerManager.deleteAll(c, TestContext.get().getUser()); + + File file1 = svc.getFileRoot(c); + if (NetworkDrive.exists(file1)) + { + FileUtil.deleteDir(file1); + } + } + } + } +} From f51bd73586be8bc639ff0641c79744214b979700 Mon Sep 17 00:00:00 2001 From: labkey-susanh Date: Thu, 8 Jan 2026 17:48:09 -0800 Subject: [PATCH 03/14] Working toward adding reference count column in file tables --- .../api/files/TableUpdaterFileListener.java | 4 +- .../experiment/FileLinkFileListener.java | 5 +- .../experiment/api/ExpDataTableImpl.java | 10 ++++ .../experiment/api/ExpFilesTableImpl.java | 52 +++++++++++++++++++ 4 files changed, 69 insertions(+), 2 deletions(-) diff --git a/api/src/org/labkey/api/files/TableUpdaterFileListener.java b/api/src/org/labkey/api/files/TableUpdaterFileListener.java index 1e3233a82f0..5b9d5dcda33 100644 --- a/api/src/org/labkey/api/files/TableUpdaterFileListener.java +++ b/api/src/org/labkey/api/files/TableUpdaterFileListener.java @@ -356,7 +356,7 @@ public SQLFragment listFilesQuery() return listFilesQuery(false, null, false); } - public SQLFragment listFilesQuery(boolean skipCreatedModified, String filePath, boolean extractName) + public SQLFragment listFilesQuery(boolean skipCreatedModified, CharSequence filePath, boolean extractName) { SQLFragment selectFrag = new SQLFragment(); selectFrag.append("SELECT\n"); @@ -418,6 +418,8 @@ else if (_table.getColumn("Folder") != null) if (StringUtils.isEmpty(filePath)) selectFrag.append(" IS NOT NULL\n"); + else if (filePath instanceof SQLFragment) + selectFrag.append(" = ").append(filePath).append("\n"); else selectFrag.append(" = ").appendStringLiteral(filePath, _table.getSchema().getSqlDialect()).append("\n"); diff --git a/experiment/src/org/labkey/experiment/FileLinkFileListener.java b/experiment/src/org/labkey/experiment/FileLinkFileListener.java index d0b538ef145..2b93db1a588 100644 --- a/experiment/src/org/labkey/experiment/FileLinkFileListener.java +++ b/experiment/src/org/labkey/experiment/FileLinkFileListener.java @@ -275,7 +275,7 @@ public SQLFragment listFilesQuery(boolean skipCreatedModified) return listFilesQuery(skipCreatedModified, null); } - public SQLFragment listFilesQuery(boolean skipCreatedModified, String filePath) + public SQLFragment listFilesQuery(boolean skipCreatedModified, CharSequence filePath) { final SQLFragment frag = new SQLFragment(); @@ -298,8 +298,11 @@ public SQLFragment listFilesQuery(boolean skipCreatedModified, String filePath) frag.append("WHERE\n"); if (StringUtils.isEmpty(filePath)) frag.append(" op.StringValue IS NOT NULL AND\n"); + else if (filePath instanceof SQLFragment) + frag.append(" op.StringValue = ").append(filePath).append(" AND\n"); else frag.append(" op.StringValue = ").appendStringLiteral(filePath, OntologyManager.getTinfoObject().getSqlDialect()).append(" AND\n"); + frag.append(" o.ObjectId = op.ObjectId AND\n"); frag.append(" PropertyId IN (\n"); frag.append(" SELECT PropertyId\n"); diff --git a/experiment/src/org/labkey/experiment/api/ExpDataTableImpl.java b/experiment/src/org/labkey/experiment/api/ExpDataTableImpl.java index 1896b1eb5b6..42fdec657a2 100644 --- a/experiment/src/org/labkey/experiment/api/ExpDataTableImpl.java +++ b/experiment/src/org/labkey/experiment/api/ExpDataTableImpl.java @@ -84,6 +84,7 @@ import org.labkey.api.writer.HtmlWriter; import org.labkey.api.writer.MemoryVirtualFile; import org.labkey.api.writer.VirtualFile; +import org.labkey.experiment.FileLinkFileListener; import org.labkey.experiment.controllers.exp.ExperimentController; import org.labkey.experiment.lineage.LineageMethod; import org.springframework.beans.MutablePropertyValues; @@ -200,6 +201,7 @@ public List addFileColumns(boolean isFilesTable) addColumn(Column.FileExtension); addColumn(Column.WebDavUrl); addColumn(Column.WebDavUrlRelative); + addColumn(getFileLinkReferenceCountColumn()); var flagCol = addColumn(Column.Flag); if (isFilesTable) flagCol.setLabel("Description"); @@ -227,6 +229,14 @@ public List addFileColumns(boolean isFilesTable) return customProps; } + private MutableColumnInfo getFileLinkReferenceCountColumn() + { + FileLinkFileListener fileListener = new FileLinkFileListener(); + SQLFragment unionSql = fileListener.listFilesQuery(true, new SQLFragment(ExprColumn.STR_TABLE_ALIAS + ".DataFileUrl")); + SQLFragment countSql = new SQLFragment("(SELECT COUNT(*) FROM (").append(unionSql).append("))"); + return new ExprColumn(this, FieldKey.fromParts("ReferenceCount"), countSql, JdbcType.INTEGER); + } + @Override public boolean supportTableRules() // intentional override { diff --git a/experiment/src/org/labkey/experiment/api/ExpFilesTableImpl.java b/experiment/src/org/labkey/experiment/api/ExpFilesTableImpl.java index db6db57f9b7..2df1a77dc1d 100644 --- a/experiment/src/org/labkey/experiment/api/ExpFilesTableImpl.java +++ b/experiment/src/org/labkey/experiment/api/ExpFilesTableImpl.java @@ -19,15 +19,18 @@ import org.jetbrains.annotations.NotNull; import org.labkey.api.data.CompareType; import org.labkey.api.data.Container; +import org.labkey.api.data.CoreSchema; import org.labkey.api.data.JdbcType; import org.labkey.api.data.MutableColumnInfo; import org.labkey.api.data.RenderContext; import org.labkey.api.data.SQLFragment; import org.labkey.api.data.SimpleFilter; +import org.labkey.api.data.SqlSelector; import org.labkey.api.exp.api.ExpData; import org.labkey.api.exp.query.ExpSchema; import org.labkey.api.files.FileContentService; import org.labkey.api.query.DetailsURL; +import org.labkey.api.query.ExprColumn; import org.labkey.api.query.FieldKey; import org.labkey.api.query.UserSchema; import org.labkey.api.security.SecurityManager; @@ -37,6 +40,7 @@ import org.labkey.api.util.InputBuilder; import org.labkey.api.view.ActionURL; import org.labkey.api.writer.HtmlWriter; +import org.labkey.experiment.FileLinkFileListener; import org.labkey.experiment.controllers.exp.ExperimentController; import java.util.ArrayList; @@ -112,6 +116,8 @@ protected void populateColumns() getMutableColumn(Column.Name).setURL(detailsURL); ActionURL deleteUrl = ExperimentController.ExperimentUrlsImpl.get().getDeleteDatasURL(getContainer(), null); setDeleteURL(new DetailsURL(deleteUrl)); + + addColumn(getFileLinkReferenceCountColumn()); } public void setDefaultColumns(List customProps) @@ -222,6 +228,52 @@ protected Object getJsonValue(ExpData data) return result; } + private MutableColumnInfo getFileLinkReferenceCountColumn() + { +// FileLinkFileListener fileListener = new FileLinkFileListener(); +// SQLFragment unionSql = fileListener.listFilesQuery(true, new SQLFragment(ExprColumn.STR_TABLE_ALIAS + ".DataFileUrl")); +// return new ExprColumn(this, FieldKey.fromParts("ReferenceCount"), unionSql, JdbcType.INTEGER); + var result = wrapColumn("ReferenceCountDisplay", _rootTable.getColumn("RowId")); + result.setJdbcType(JdbcType.VARCHAR); + result.setDisplayColumnFactory(colInfo -> new ExpDataFileColumn(colInfo) + { + @Override + protected void renderData(HtmlWriter out, ExpData data) + { + + if (data == null || StringUtils.isEmpty(data.getDataFileUrl())) + out.write(""); + else + { + FileLinkFileListener fileListener = new FileLinkFileListener(); + SQLFragment unionSql = fileListener.listFilesQuery(true, data.getFile().getAbsolutePath()); + + long count = new SqlSelector(CoreSchema.getInstance().getSchema(), unionSql).getRowCount(); + + out.write(count); + } + } + + @Override + protected Object getJsonValue(ExpData data) + { + Object val; + if (data == null || StringUtils.isEmpty(data.getDataFileUrl())) + val = null; + else + { + FileLinkFileListener fileListener = new FileLinkFileListener(); + SQLFragment unionSql = fileListener.listFilesQuery(true, data.getFile().getAbsolutePath()); + + val = new SqlSelector(CoreSchema.getInstance().getSchema(), unionSql).getRowCount(); + + } + return val; + } + }); + return result; + } + @Override public boolean hasPermission(@NotNull UserPrincipal user, @NotNull Class perm) { From 7350ecc75b66e45d6b4b1858c4897f2412005301 Mon Sep 17 00:00:00 2001 From: XingY Date: Fri, 9 Jan 2026 09:26:34 -0800 Subject: [PATCH 04/14] experimental flag --- .../org/labkey/api/exp/query/ExpSchema.java | 7 + .../labkey/experiment/ExperimentModule.java | 2244 +++++++++-------- 2 files changed, 1131 insertions(+), 1120 deletions(-) diff --git a/api/src/org/labkey/api/exp/query/ExpSchema.java b/api/src/org/labkey/api/exp/query/ExpSchema.java index 62b4ba7b661..1129bc84fda 100644 --- a/api/src/org/labkey/api/exp/query/ExpSchema.java +++ b/api/src/org/labkey/api/exp/query/ExpSchema.java @@ -47,6 +47,7 @@ import org.labkey.api.security.User; import org.labkey.api.security.permissions.InsertPermission; import org.labkey.api.security.permissions.ReadPermission; +import org.labkey.api.settings.AppProps; import org.labkey.api.util.StringExpression; import org.labkey.api.view.ActionURL; import org.labkey.api.view.ViewContext; @@ -228,6 +229,12 @@ public TableInfo createTable(ExpSchema expSchema, String queryName, ContainerFil { return new ExpStaleSampleFilesTable(expSchema, cf); } + + @Override + public boolean includeTable() + { + return AppProps.getInstance().isOptionalFeatureEnabled(SAMPLE_FILES_TABLE); + } }, SampleStatus { diff --git a/experiment/src/org/labkey/experiment/ExperimentModule.java b/experiment/src/org/labkey/experiment/ExperimentModule.java index 435448ca0b7..f29a5ab73b2 100644 --- a/experiment/src/org/labkey/experiment/ExperimentModule.java +++ b/experiment/src/org/labkey/experiment/ExperimentModule.java @@ -1,1120 +1,1124 @@ -/* - * Copyright (c) 2008-2019 LabKey Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.labkey.experiment; - -import org.apache.commons.lang3.math.NumberUtils; -import org.jetbrains.annotations.NotNull; -import org.jetbrains.annotations.Nullable; -import org.labkey.api.admin.FolderSerializationRegistry; -import org.labkey.api.assay.AssayProvider; -import org.labkey.api.assay.AssayService; -import org.labkey.api.attachments.AttachmentService; -import org.labkey.api.audit.AuditLogService; -import org.labkey.api.audit.SampleTimelineAuditEvent; -import org.labkey.api.collections.LongHashMap; -import org.labkey.api.data.Container; -import org.labkey.api.data.ContainerFilter; -import org.labkey.api.data.ContainerManager; -import org.labkey.api.data.CoreSchema; -import org.labkey.api.data.DbSchema; -import org.labkey.api.data.DbSchemaType; -import org.labkey.api.data.JdbcType; -import org.labkey.api.data.NameGenerator; -import org.labkey.api.data.SQLFragment; -import org.labkey.api.data.SimpleFilter; -import org.labkey.api.data.SimpleFilter.FilterClause; -import org.labkey.api.data.SqlSelector; -import org.labkey.api.data.TableInfo; -import org.labkey.api.data.TableSelector; -import org.labkey.api.data.UpgradeCode; -import org.labkey.api.defaults.DefaultValueService; -import org.labkey.api.exp.ExperimentException; -import org.labkey.api.exp.ExperimentRunType; -import org.labkey.api.exp.Lsid; -import org.labkey.api.exp.OntologyManager; -import org.labkey.api.exp.PropertyType; -import org.labkey.api.exp.api.DefaultExperimentDataHandler; -import org.labkey.api.exp.api.ExpData; -import org.labkey.api.exp.api.ExpDataClass; -import org.labkey.api.exp.api.ExpLineageService; -import org.labkey.api.exp.api.ExpMaterial; -import org.labkey.api.exp.api.ExpProtocol; -import org.labkey.api.exp.api.ExpProtocolAttachmentType; -import org.labkey.api.exp.api.ExpRunAttachmentType; -import org.labkey.api.exp.api.ExpSampleType; -import org.labkey.api.exp.api.ExperimentJSONConverter; -import org.labkey.api.exp.api.ExperimentService; -import org.labkey.api.exp.api.FilterProtocolInputCriteria; -import org.labkey.api.exp.api.SampleTypeDomainKind; -import org.labkey.api.exp.api.SampleTypeService; -import org.labkey.api.exp.api.StorageProvisioner; -import org.labkey.api.exp.property.DomainAuditProvider; -import org.labkey.api.exp.property.DomainPropertyAuditProvider; -import org.labkey.api.exp.property.ExperimentProperty; -import org.labkey.api.exp.property.PropertyService; -import org.labkey.api.exp.property.SystemProperty; -import org.labkey.api.exp.query.ExpDataClassTable; -import org.labkey.api.exp.query.ExpSampleTypeTable; -import org.labkey.api.exp.query.ExpSchema; -import org.labkey.api.exp.query.SamplesSchema; -import org.labkey.api.exp.xar.LSIDRelativizer; -import org.labkey.api.exp.xar.LsidUtils; -import org.labkey.api.files.FileContentService; -import org.labkey.api.files.TableUpdaterFileListener; -import org.labkey.api.migration.DatabaseMigrationService; -import org.labkey.api.migration.ExperimentDeleteService; -import org.labkey.api.migration.MigrationTableHandler; -import org.labkey.api.module.ModuleContext; -import org.labkey.api.module.ModuleLoader; -import org.labkey.api.module.SpringModule; -import org.labkey.api.module.Summary; -import org.labkey.api.ontology.OntologyService; -import org.labkey.api.ontology.Quantity; -import org.labkey.api.ontology.Unit; -import org.labkey.api.pipeline.PipelineService; -import org.labkey.api.query.FieldKey; -import org.labkey.api.query.FilteredTable; -import org.labkey.api.query.QueryService; -import org.labkey.api.query.UserSchema; -import org.labkey.api.search.SearchService; -import org.labkey.api.security.User; -import org.labkey.api.security.roles.RoleManager; -import org.labkey.api.settings.AppProps; -import org.labkey.api.settings.OptionalFeatureService; -import org.labkey.api.usageMetrics.UsageMetricsService; -import org.labkey.api.util.GUID; -import org.labkey.api.util.JspTestCase; -import org.labkey.api.util.PageFlowUtil; -import org.labkey.api.util.StringUtilsLabKey; -import org.labkey.api.util.SystemMaintenance; -import org.labkey.api.view.AlwaysAvailableWebPartFactory; -import org.labkey.api.view.BaseWebPartFactory; -import org.labkey.api.view.HttpView; -import org.labkey.api.view.JspView; -import org.labkey.api.view.Portal; -import org.labkey.api.view.ViewContext; -import org.labkey.api.view.WebPartFactory; -import org.labkey.api.view.WebPartView; -import org.labkey.api.view.template.WarningService; -import org.labkey.api.vocabulary.security.DesignVocabularyPermission; -import org.labkey.api.webdav.WebdavResource; -import org.labkey.api.webdav.WebdavService; -import org.labkey.experiment.api.DataClassDomainKind; -import org.labkey.experiment.api.ExpDataClassImpl; -import org.labkey.experiment.api.ExpDataClassTableImpl; -import org.labkey.experiment.api.ExpDataClassType; -import org.labkey.experiment.api.ExpDataImpl; -import org.labkey.experiment.api.ExpDataTableImpl; -import org.labkey.experiment.api.ExpMaterialImpl; -import org.labkey.experiment.api.ExpProtocolImpl; -import org.labkey.experiment.api.ExpSampleTypeImpl; -import org.labkey.experiment.api.ExpSampleTypeTableImpl; -import org.labkey.experiment.api.ExperimentServiceImpl; -import org.labkey.experiment.api.ExperimentStressTest; -import org.labkey.experiment.api.GraphAlgorithms; -import org.labkey.experiment.api.LineageTest; -import org.labkey.experiment.api.LogDataType; -import org.labkey.experiment.api.Protocol; -import org.labkey.experiment.api.SampleTypeServiceImpl; -import org.labkey.experiment.api.UniqueValueCounterTestCase; -import org.labkey.experiment.api.VocabularyDomainKind; -import org.labkey.experiment.api.data.ChildOfCompareType; -import org.labkey.experiment.api.data.ChildOfMethod; -import org.labkey.experiment.api.data.LineageCompareType; -import org.labkey.experiment.api.data.ParentOfCompareType; -import org.labkey.experiment.api.data.ParentOfMethod; -import org.labkey.experiment.api.property.DomainImpl; -import org.labkey.experiment.api.property.DomainPropertyImpl; -import org.labkey.experiment.api.property.LengthValidator; -import org.labkey.experiment.api.property.LookupValidator; -import org.labkey.experiment.api.property.PropertyServiceImpl; -import org.labkey.experiment.api.property.RangeValidator; -import org.labkey.experiment.api.property.RegExValidator; -import org.labkey.experiment.api.property.StorageNameGenerator; -import org.labkey.experiment.api.property.StorageProvisionerImpl; -import org.labkey.experiment.api.property.TextChoiceValidator; -import org.labkey.experiment.controllers.exp.ExperimentController; -import org.labkey.experiment.controllers.property.PropertyController; -import org.labkey.experiment.defaults.DefaultValueServiceImpl; -import org.labkey.experiment.lineage.ExpLineageServiceImpl; -import org.labkey.experiment.lineage.LineagePerfTest; -import org.labkey.experiment.pipeline.ExperimentPipelineProvider; -import org.labkey.experiment.pipeline.XarTestPipelineJob; -import org.labkey.experiment.samples.DataClassFolderImporter; -import org.labkey.experiment.samples.DataClassFolderWriter; -import org.labkey.experiment.samples.SampleStatusFolderImporter; -import org.labkey.experiment.samples.SampleTimelineAuditProvider; -import org.labkey.experiment.samples.SampleTypeFolderImporter; -import org.labkey.experiment.samples.SampleTypeFolderWriter; -import org.labkey.experiment.security.DataClassDesignerRole; -import org.labkey.experiment.security.SampleTypeDesignerRole; -import org.labkey.experiment.types.TypesController; -import org.labkey.experiment.xar.FolderXarImporterFactory; -import org.labkey.experiment.xar.FolderXarWriterFactory; - -import java.util.ArrayList; -import java.util.Collection; -import java.util.Collections; -import java.util.HashMap; -import java.util.HashSet; -import java.util.LinkedHashSet; -import java.util.LinkedList; -import java.util.List; -import java.util.Map; -import java.util.Set; -import java.util.function.Supplier; -import java.util.stream.Collectors; - -import static org.labkey.api.data.ColumnRenderPropertiesImpl.STORAGE_UNIQUE_ID_CONCEPT_URI; -import static org.labkey.api.data.ColumnRenderPropertiesImpl.TEXT_CHOICE_CONCEPT_URI; -import static org.labkey.api.exp.api.ExperimentService.MODULE_NAME; - -public class ExperimentModule extends SpringModule -{ - private static final String SAMPLE_TYPE_WEB_PART_NAME = "Sample Types"; - private static final String PROTOCOL_WEB_PART_NAME = "Protocols"; - - public static final String AMOUNT_AND_UNIT_UPGRADE_PROP = "AmountAndUnitAudit"; - public static final String TRANSACTION_ID_PROP = "AuditTransactionId"; - public static final String AUDIT_COUNT_PROP = "AuditRecordCount"; - public static final String EXPERIMENT_RUN_WEB_PART_NAME = "Experiment Runs"; - - @Override - public String getName() - { - return MODULE_NAME; - } - - @Override - public Double getSchemaVersion() - { - return 25.013; - } - - @Nullable - @Override - public UpgradeCode getUpgradeCode() - { - return new ExperimentUpgradeCode(); - } - - @Override - protected void init() - { - addController("experiment", ExperimentController.class); - addController("experiment-types", TypesController.class); - addController("property", PropertyController.class); - ExperimentService.setInstance(new ExperimentServiceImpl()); - SampleTypeService.setInstance(new SampleTypeServiceImpl()); - DefaultValueService.setInstance(new DefaultValueServiceImpl()); - StorageProvisioner.setInstance(StorageProvisionerImpl.get()); - ExpLineageService.setInstance(new ExpLineageServiceImpl()); - - PropertyServiceImpl propertyServiceImpl = new PropertyServiceImpl(); - PropertyService.setInstance(propertyServiceImpl); - UsageMetricsService.get().registerUsageMetrics(getName(), propertyServiceImpl); - - UsageMetricsService.get().registerUsageMetrics(getName(), FileLinkMetricsProvider.getInstance()); - - ExperimentProperty.register(); - SamplesSchema.register(this); - ExpSchema.register(this); - - PropertyService.get().registerDomainKind(new SampleTypeDomainKind()); - PropertyService.get().registerDomainKind(new DataClassDomainKind()); - PropertyService.get().registerDomainKind(new VocabularyDomainKind()); - - QueryService.get().addCompareType(new ChildOfCompareType()); - QueryService.get().addCompareType(new ParentOfCompareType()); - QueryService.get().addCompareType(new LineageCompareType()); - QueryService.get().registerMethod(ChildOfMethod.NAME, new ChildOfMethod(), JdbcType.BOOLEAN, 2, 3); - QueryService.get().registerMethod(ParentOfMethod.NAME, new ParentOfMethod(), JdbcType.BOOLEAN, 2, 3); - QueryService.get().addQueryListener(new ExperimentQueryChangeListener()); - QueryService.get().addQueryListener(new PropertyQueryChangeListener()); - - PropertyService.get().registerValidatorKind(new RegExValidator()); - PropertyService.get().registerValidatorKind(new RangeValidator()); - PropertyService.get().registerValidatorKind(new LookupValidator()); - PropertyService.get().registerValidatorKind(new LengthValidator()); - PropertyService.get().registerValidatorKind(new TextChoiceValidator()); - - ExperimentService.get().registerExperimentDataHandler(new DefaultExperimentDataHandler()); - ExperimentService.get().registerProtocolInputCriteria(new FilterProtocolInputCriteria.Factory()); - ExperimentService.get().registerNameExpressionType("sampletype", "exp", "MaterialSource", "nameexpression"); - ExperimentService.get().registerNameExpressionType("aliquots", "exp", "MaterialSource", "aliquotnameexpression"); - ExperimentService.get().registerNameExpressionType("dataclass", "exp", "DataClass", "nameexpression"); - - OptionalFeatureService.get().addExperimentalFeatureFlag(AppProps.EXPERIMENTAL_RESOLVE_PROPERTY_URI_COLUMNS, "Resolve property URIs as columns on experiment tables", - "If a column is not found on an experiment table, attempt to resolve the column name as a Property URI and add it as a property column", false); - if (CoreSchema.getInstance().getSqlDialect().isSqlServer()) - { - OptionalFeatureService.get().addExperimentalFeatureFlag(NameGenerator.EXPERIMENTAL_WITH_COUNTER, "Use strict incremental withCounter and rootSampleCount expression", - "When withCounter or rootSampleCount is used in name expression, make sure the count increments one-by-one and does not jump.", true); - } - else - { - OptionalFeatureService.get().addExperimentalFeatureFlag(NameGenerator.EXPERIMENTAL_ALLOW_GAP_COUNTER, "Allow gap with withCounter and rootSampleCount expression", - "Check this option if gaps in the count generated by withCounter or rootSampleCount name expression are allowed.", true); - } - OptionalFeatureService.get().addExperimentalFeatureFlag(AppProps.QUANTITY_COLUMN_SUFFIX_TESTING, "Quantity column suffix testing", - "If a column name contains a \"__\" suffix, this feature allows for testing it as a Quantity display column", false); - - RoleManager.registerPermission(new DesignVocabularyPermission(), true); - RoleManager.registerRole(new SampleTypeDesignerRole()); - RoleManager.registerRole(new DataClassDesignerRole()); - - AttachmentService.get().registerAttachmentType(ExpRunAttachmentType.get()); - AttachmentService.get().registerAttachmentType(ExpProtocolAttachmentType.get()); - - WebdavService.get().addExpDataProvider((path, container) -> ExperimentService.get().getAllExpDataByURL(path, container)); - ExperimentService.get().registerObjectReferencer(ExperimentServiceImpl.get()); - - addModuleProperty(new LineageMaximumDepthModuleProperty(this)); - WarningService.get().register(new ExperimentWarningProvider()); - } - - @Override - public boolean hasScripts() - { - return true; - } - - @Override - @NotNull - protected Collection createWebPartFactories() - { - List result = new ArrayList<>(); - - BaseWebPartFactory runGroupsFactory = new BaseWebPartFactory(RunGroupWebPart.WEB_PART_NAME, WebPartFactory.LOCATION_BODY, WebPartFactory.LOCATION_RIGHT) - { - @Override - public WebPartView getWebPartView(@NotNull ViewContext portalCtx, @NotNull Portal.WebPart webPart) - { - return new RunGroupWebPart(portalCtx, WebPartFactory.LOCATION_RIGHT.equalsIgnoreCase(webPart.getLocation()), webPart); - } - }; - runGroupsFactory.addLegacyNames("Experiments", "Experiment", "Experiment Navigator", "Narrow Experiments"); - result.add(runGroupsFactory); - - BaseWebPartFactory runTypesFactory = new BaseWebPartFactory(RunTypeWebPart.WEB_PART_NAME, WebPartFactory.LOCATION_BODY, WebPartFactory.LOCATION_RIGHT) - { - @Override - public WebPartView getWebPartView(@NotNull ViewContext portalCtx, @NotNull Portal.WebPart webPart) - { - return new RunTypeWebPart(); - } - }; - result.add(runTypesFactory); - - result.add(new ExperimentRunWebPartFactory()); - BaseWebPartFactory sampleTypeFactory = new BaseWebPartFactory(SAMPLE_TYPE_WEB_PART_NAME, WebPartFactory.LOCATION_BODY, WebPartFactory.LOCATION_RIGHT) - { - @Override - public WebPartView getWebPartView(@NotNull ViewContext portalCtx, @NotNull Portal.WebPart webPart) - { - return new SampleTypeWebPart(WebPartFactory.LOCATION_RIGHT.equalsIgnoreCase(webPart.getLocation()), portalCtx); - } - }; - sampleTypeFactory.addLegacyNames("Narrow Sample Sets", "Sample Sets"); - result.add(sampleTypeFactory); - result.add(new AlwaysAvailableWebPartFactory("Samples Menu", false, false, WebPartFactory.LOCATION_MENUBAR) { - @Override - public WebPartView getWebPartView(@NotNull ViewContext portalCtx, @NotNull Portal.WebPart webPart) - { - WebPartView view = new JspView<>("/org/labkey/experiment/samplesAndAnalytes.jsp", webPart); - view.setTitle("Samples"); - return view; - } - }); - - result.add(new AlwaysAvailableWebPartFactory("Data Classes", false, false, WebPartFactory.LOCATION_BODY, WebPartFactory.LOCATION_RIGHT) { - @Override - public WebPartView getWebPartView(@NotNull ViewContext portalCtx, @NotNull Portal.WebPart webPart) - { - return new DataClassWebPart(WebPartFactory.LOCATION_RIGHT.equalsIgnoreCase(webPart.getLocation()), portalCtx, webPart); - } - }); - - BaseWebPartFactory narrowProtocolFactory = new BaseWebPartFactory(PROTOCOL_WEB_PART_NAME, WebPartFactory.LOCATION_RIGHT) - { - @Override - public WebPartView getWebPartView(@NotNull ViewContext portalCtx, @NotNull Portal.WebPart webPart) - { - return new ProtocolWebPart(WebPartFactory.LOCATION_RIGHT.equalsIgnoreCase(webPart.getLocation()), portalCtx); - } - }; - narrowProtocolFactory.addLegacyNames("Narrow Protocols"); - result.add(narrowProtocolFactory); - - return result; - } - - private void addDataResourceResolver(String categoryName) - { - SearchService.get().addResourceResolver(categoryName, new SearchService.ResourceResolver() - { - @Override - public WebdavResource resolve(@NotNull String resourceIdentifier) - { - ExpDataImpl data = ExpDataImpl.fromDocumentId(resourceIdentifier); - if (data == null) - return null; - - return data.createIndexDocument(null); - } - - @Override - public Map getCustomSearchJson(User user, @NotNull String resourceIdentifier) - { - ExpDataImpl data = ExpDataImpl.fromDocumentId(resourceIdentifier); - if (data == null) - return null; - - return ExperimentJSONConverter.serializeData(data, user, ExperimentJSONConverter.DEFAULT_SETTINGS).toMap(); - } - - @Override - public Map> getCustomSearchJsonMap(User user, @NotNull Collection resourceIdentifiers) - { - Map idDataMap = ExpDataImpl.fromDocumentIds(resourceIdentifiers); - if (idDataMap == null) - return null; - - Map> searchJsonMap = new HashMap<>(); - for (String resourceIdentifier : idDataMap.keySet()) - searchJsonMap.put(resourceIdentifier, ExperimentJSONConverter.serializeData(idDataMap.get(resourceIdentifier), user, ExperimentJSONConverter.DEFAULT_SETTINGS).toMap()); - return searchJsonMap; - } - }); - } - - private void addDataClassResourceResolver(String categoryName) - { - SearchService.get().addResourceResolver(categoryName, new SearchService.ResourceResolver(){ - @Override - public Map getCustomSearchJson(User user, @NotNull String resourceIdentifier) - { - int rowId = NumberUtils.toInt(resourceIdentifier.replace(categoryName + ":", "")); - if (rowId == 0) - return null; - - ExpDataClass dataClass = ExperimentService.get().getDataClass(rowId); - if (dataClass == null) - return null; - - Map properties = ExperimentJSONConverter.serializeExpObject(dataClass, null, ExperimentJSONConverter.DEFAULT_SETTINGS, user).toMap(); - - //Need to map to proper Icon - properties.put("type", "dataClass" + (dataClass.getCategory() != null ? ":" + dataClass.getCategory() : "")); - - return properties; - } - }); - } - - private void addSampleTypeResourceResolver(String categoryName) - { - SearchService.get().addResourceResolver(categoryName, new SearchService.ResourceResolver(){ - @Override - public Map getCustomSearchJson(User user, @NotNull String resourceIdentifier) - { - int rowId = NumberUtils.toInt(resourceIdentifier.replace(categoryName + ":", "")); - if (rowId == 0) - return null; - - ExpSampleType sampleType = SampleTypeService.get().getSampleType(rowId); - if (sampleType == null) - return null; - - Map properties = ExperimentJSONConverter.serializeExpObject(sampleType, null, ExperimentJSONConverter.DEFAULT_SETTINGS, user).toMap(); - - //Need to map to proper Icon - properties.put("type", "sampleSet"); - - return properties; - } - }); - } - - private void addSampleResourceResolver(String categoryName) - { - SearchService.get().addResourceResolver(categoryName, new SearchService.ResourceResolver(){ - @Override - public Map getCustomSearchJson(User user, @NotNull String resourceIdentifier) - { - int rowId = NumberUtils.toInt(resourceIdentifier.replace(categoryName + ":", "")); - if (rowId == 0) - return null; - - ExpMaterial material = ExperimentService.get().getExpMaterial(rowId); - if (material == null) - return null; - - return ExperimentJSONConverter.serializeMaterial(material, user, ExperimentJSONConverter.DEFAULT_SETTINGS).toMap(); - } - - @Override - public Map> getCustomSearchJsonMap(User user, @NotNull Collection resourceIdentifiers) - { - Set rowIds = new HashSet<>(); - Map rowIdIdentifierMap = new LongHashMap<>(); - for (String resourceIdentifier : resourceIdentifiers) - { - long rowId = NumberUtils.toLong(resourceIdentifier.replace(categoryName + ":", "")); - if (rowId != 0) - { - rowIds.add(rowId); - rowIdIdentifierMap.put(rowId, resourceIdentifier); - } - } - - Map> searchJsonMap = new HashMap<>(); - for (ExpMaterial material : ExperimentService.get().getExpMaterials(rowIds)) - { - searchJsonMap.put( - rowIdIdentifierMap.get(material.getRowId()), - ExperimentJSONConverter.serializeMaterial(material, user, ExperimentJSONConverter.DEFAULT_SETTINGS).toMap() - ); - } - - return searchJsonMap; - } - }); - } - - @Override - protected void startupAfterSpringConfig(ModuleContext moduleContext) - { - SearchService ss = SearchService.get(); -// ss.addSearchCategory(OntologyManager.conceptCategory); - ss.addSearchCategory(ExpSampleTypeImpl.searchCategory); - ss.addSearchCategory(ExpSampleTypeImpl.mediaSearchCategory); - ss.addSearchCategory(ExpMaterialImpl.searchCategory); - ss.addSearchCategory(ExpMaterialImpl.mediaSearchCategory); - ss.addSearchCategory(ExpDataClassImpl.SEARCH_CATEGORY); - ss.addSearchCategory(ExpDataClassImpl.MEDIA_SEARCH_CATEGORY); - ss.addSearchCategory(ExpDataImpl.expDataCategory); - ss.addSearchCategory(ExpDataImpl.expMediaDataCategory); - ss.addSearchResultTemplate(new ExpDataImpl.DataSearchResultTemplate()); - addDataResourceResolver(ExpDataImpl.expDataCategory.getName()); - addDataResourceResolver(ExpDataImpl.expMediaDataCategory.getName()); - addDataClassResourceResolver(ExpDataClassImpl.SEARCH_CATEGORY.getName()); - addDataClassResourceResolver(ExpDataClassImpl.MEDIA_SEARCH_CATEGORY.getName()); - addSampleTypeResourceResolver(ExpSampleTypeImpl.searchCategory.getName()); - addSampleTypeResourceResolver(ExpSampleTypeImpl.mediaSearchCategory.getName()); - addSampleResourceResolver(ExpMaterialImpl.searchCategory.getName()); - addSampleResourceResolver(ExpMaterialImpl.mediaSearchCategory.getName()); - ss.addDocumentProvider(ExperimentServiceImpl.get()); - - PipelineService.get().registerPipelineProvider(new ExperimentPipelineProvider(this)); - ExperimentService.get().registerExperimentRunTypeSource(container -> Collections.singleton(ExperimentRunType.ALL_RUNS_TYPE)); - ExperimentService.get().registerDataType(new LogDataType()); - - AuditLogService.get().registerAuditType(new DomainAuditProvider()); - AuditLogService.get().registerAuditType(new DomainPropertyAuditProvider()); - AuditLogService.get().registerAuditType(new ExperimentAuditProvider()); - AuditLogService.get().registerAuditType(new SampleTypeAuditProvider()); - AuditLogService.get().registerAuditType(new SampleTimelineAuditProvider()); - - FileContentService fileContentService = FileContentService.get(); - if (null != fileContentService) - { - fileContentService.addFileListener(new ExpDataFileListener()); - fileContentService.addFileListener(new TableUpdaterFileListener(ExperimentService.get().getTinfoExperimentRun(), "FilePathRoot", TableUpdaterFileListener.Type.fileRootPath, "RowId")); - fileContentService.addFileListener(new FileLinkFileListener()); - } - ContainerManager.addContainerListener(new ContainerManager.ContainerListener() - { - @Override - public void containerDeleted(Container c, User user) - { - try - { - ExperimentService.get().deleteAllExpObjInContainer(c, user); - } - catch (ExperimentException ee) - { - throw new RuntimeException(ee); - } - } - }, - // This is in the Last group because when a container is deleted, - // the Experiment listener needs to be called after the Study listener, - // because Study needs the metadata held by Experiment to delete properly. - // but it should be before the CoreContainerListener - ContainerManager.ContainerListener.Order.Last); - - if (ModuleLoader.getInstance().shouldInsertData()) - SystemProperty.registerProperties(); - - FolderSerializationRegistry folderRegistry = FolderSerializationRegistry.get(); - if (null != folderRegistry) - { - folderRegistry.addFactories(new FolderXarWriterFactory(), new FolderXarImporterFactory()); - folderRegistry.addWriterFactory(new SampleTypeFolderWriter.SampleTypeDesignWriter.Factory()); - folderRegistry.addWriterFactory(new SampleTypeFolderWriter.SampleTypeDataWriter.Factory()); - folderRegistry.addWriterFactory(new DataClassFolderWriter.DataClassDesignWriter.Factory()); - folderRegistry.addWriterFactory(new DataClassFolderWriter.DataClassDataWriter.Factory()); - folderRegistry.addImportFactory(new SampleTypeFolderImporter.Factory()); - folderRegistry.addImportFactory(new DataClassFolderImporter.Factory()); - folderRegistry.addImportFactory(new SampleStatusFolderImporter.Factory()); - } - - AttachmentService.get().registerAttachmentType(ExpDataClassType.get()); - - WebdavService.get().addProvider(new ScriptsResourceProvider()); - - SystemMaintenance.addTask(new FileLinkMetricsMaintenanceTask()); - - UsageMetricsService svc = UsageMetricsService.get(); - if (null != svc) - { - svc.registerUsageMetrics(getName(), () -> { - Map results = new HashMap<>(); - - DbSchema schema = ExperimentService.get().getSchema(); - if (AssayService.get() != null) - { - Map assayMetrics = new HashMap<>(); - SQLFragment baseRunSQL = new SQLFragment("SELECT COUNT(*) FROM ").append(ExperimentService.get().getTinfoExperimentRun(), "r").append(" WHERE lsid LIKE ?"); - SQLFragment baseProtocolSQL = new SQLFragment("SELECT * FROM ").append(ExperimentService.get().getTinfoProtocol(), "p").append(" WHERE lsid LIKE ? AND ApplicationType = ?"); - for (AssayProvider assayProvider : AssayService.get().getAssayProviders()) - { - Map protocolMetrics = new HashMap<>(); - - // Run count across all assay designs of this type - SQLFragment runSQL = new SQLFragment(baseRunSQL); - runSQL.add(Lsid.namespaceLikeString(assayProvider.getRunLSIDPrefix())); - protocolMetrics.put("runCount", new SqlSelector(schema, runSQL).getObject(Long.class)); - - // Number of assay designs of this type - SQLFragment protocolSQL = new SQLFragment(baseProtocolSQL); - protocolSQL.add(assayProvider.getProtocolPattern()); - protocolSQL.add(ExpProtocol.ApplicationType.ExperimentRun.toString()); - List protocols = new SqlSelector(schema, protocolSQL).getArrayList(Protocol.class); - protocolMetrics.put("protocolCount", protocols.size()); - - List wrappedProtocols = protocols.stream().map(ExpProtocolImpl::new).collect(Collectors.toList()); - - protocolMetrics.put("resultRowCount", assayProvider.getResultRowCount(wrappedProtocols)); - - // Primary implementation class - protocolMetrics.put("implementingClass", assayProvider.getClass()); - - assayMetrics.put(assayProvider.getName(), protocolMetrics); - } - assayMetrics.put("autoLinkedAssayCount", new SqlSelector(schema, "SELECT COUNT(*) FROM exp.protocol EP JOIN exp.objectPropertiesView OP ON EP.lsid = OP.objecturi WHERE OP.propertyuri = 'terms.labkey.org#AutoCopyTargetContainer'").getObject(Long.class)); - assayMetrics.put("protocolsWithTransformScriptCount", new SqlSelector(schema, "SELECT COUNT(*) FROM exp.protocol EP JOIN exp.objectPropertiesView OP ON EP.lsid = OP.objecturi WHERE OP.name = 'TransformScript' AND status = 'Active'").getObject(Long.class)); - assayMetrics.put("protocolsWithTransformScriptRunOnEditCount", new SqlSelector(schema, "SELECT COUNT(*) FROM exp.protocol EP JOIN exp.objectPropertiesView OP ON EP.lsid = OP.objecturi WHERE OP.name = 'TransformScript' AND status = 'Active' AND OP.stringvalue LIKE '%\"INSERT\"%'").getObject(Long.class)); - assayMetrics.put("protocolsWithTransformScriptRunOnImportCount", new SqlSelector(schema, "SELECT COUNT(*) FROM exp.protocol EP JOIN exp.objectPropertiesView OP ON EP.lsid = OP.objecturi WHERE OP.name = 'TransformScript' AND status = 'Active' AND OP.stringvalue LIKE '%\"INSERT\"%'").getObject(Long.class)); - - assayMetrics.put("standardAssayWithPlateSupportCount", new SqlSelector(schema, "SELECT COUNT(*) FROM exp.protocol EP JOIN exp.objectPropertiesView OP ON EP.lsid = OP.objecturi WHERE OP.name = 'PlateMetadata' AND floatValue = 1").getObject(Long.class)); - SQLFragment runsWithPlateSQL = new SQLFragment(""" - SELECT COUNT(*) FROM exp.experimentrun r - INNER JOIN exp.object o ON o.objectUri = r.lsid - INNER JOIN exp.objectproperty op ON op.objectId = o.objectId - WHERE op.propertyid IN ( - SELECT propertyid FROM exp.propertydescriptor WHERE name = ? AND lookupquery = ? - )"""); - assayMetrics.put("standardAssayRunsWithPlateTemplate", new SqlSelector(schema, new SQLFragment(runsWithPlateSQL).add("PlateTemplate").add("PlateTemplate")).getObject(Long.class)); - assayMetrics.put("standardAssayRunsWithPlateSet", new SqlSelector(schema, new SQLFragment(runsWithPlateSQL).add("PlateSet").add("PlateSet")).getObject(Long.class)); - - assayMetrics.put("assayRunsFileColumnCount", new SqlSelector(schema, """ - SELECT COUNT(DISTINCT DD.DomainURI) FROM - exp.PropertyDescriptor D\s - JOIN exp.PropertyDomain PD ON D.propertyId = PD.propertyid - JOIN exp.DomainDescriptor DD on PD.domainID = DD.domainId - WHERE DD.domainUri LIKE ? AND D.rangeURI = ?""", "urn:lsid:%:" + ExpProtocol.AssayDomainTypes.Run.getPrefix() + ".%", PropertyType.FILE_LINK.getTypeUri()).getObject(Long.class)); - - assayMetrics.put("assayResultsFileColumnCount", new SqlSelector(schema, """ - SELECT COUNT(DISTINCT DD.DomainURI) FROM - exp.PropertyDescriptor D\s - JOIN exp.PropertyDomain PD ON D.propertyId = PD.propertyid - JOIN exp.DomainDescriptor DD on PD.domainID = DD.domainId - WHERE DD.domainUri LIKE ? AND D.rangeURI = ?""", "urn:lsid:%:" + ExpProtocol.AssayDomainTypes.Result.getPrefix() + ".%", PropertyType.FILE_LINK.getTypeUri()).getObject(Long.class)); - - Map sampleLookupCountMetrics = new HashMap<>(); - SQLFragment baseAssaySampleLookupSQL = new SQLFragment("SELECT COUNT(*) FROM exp.propertydescriptor WHERE (lookupschema = 'samples' OR (lookupschema = 'exp' AND lookupquery = 'Materials')) AND propertyuri LIKE ?"); - - SQLFragment batchAssaySampleLookupSQL = new SQLFragment(baseAssaySampleLookupSQL); - batchAssaySampleLookupSQL.add("urn:lsid:%:" + ExpProtocol.AssayDomainTypes.Batch.getPrefix() + ".%"); - sampleLookupCountMetrics.put("batchDomain", new SqlSelector(schema, batchAssaySampleLookupSQL).getObject(Long.class)); - - SQLFragment runAssaySampleLookupSQL = new SQLFragment(baseAssaySampleLookupSQL); - runAssaySampleLookupSQL.add("urn:lsid:%:" + ExpProtocol.AssayDomainTypes.Run.getPrefix() + ".%"); - sampleLookupCountMetrics.put("runDomain", new SqlSelector(schema, runAssaySampleLookupSQL).getObject(Long.class)); - - SQLFragment resultAssaySampleLookupSQL = new SQLFragment(baseAssaySampleLookupSQL); - resultAssaySampleLookupSQL.add("urn:lsid:%:" + ExpProtocol.AssayDomainTypes.Result.getPrefix() + ".%"); - sampleLookupCountMetrics.put("resultDomain", new SqlSelector(schema, resultAssaySampleLookupSQL).getObject(Long.class)); - - SQLFragment resultAssayMultipleSampleLookupSQL = new SQLFragment( - """ - SELECT COUNT(*) FROM ( - SELECT PD.domainid, COUNT(*) AS PropCount - FROM exp.propertydescriptor D - JOIN exp.PropertyDomain PD ON D.propertyId = PD.propertyid - WHERE (lookupschema = 'samples' OR (lookupschema = 'exp' AND lookupquery = 'Materials')) - AND propertyuri LIKE ? - GROUP BY PD.domainid - ) X WHERE X.PropCount > 1""" - ); - resultAssayMultipleSampleLookupSQL.add("urn:lsid:%:" + ExpProtocol.AssayDomainTypes.Result.getPrefix() + ".%"); - sampleLookupCountMetrics.put("resultDomainWithMultiple", new SqlSelector(schema, resultAssayMultipleSampleLookupSQL).getObject(Long.class)); - - assayMetrics.put("sampleLookupCount", sampleLookupCountMetrics); - - - // Putting these metrics at the same level as the other BooleanColumnCount metrics (e.g., sampleTypeWithBooleanColumnCount) - results.put("assayResultWithBooleanColumnCount", new SqlSelector(schema, """ - SELECT COUNT(DISTINCT DD.DomainURI) FROM - exp.PropertyDescriptor D\s - JOIN exp.PropertyDomain PD ON D.propertyId = PD.propertyid - JOIN exp.DomainDescriptor DD on PD.domainID = DD.domainId - WHERE D.propertyURI LIKE ? AND D.rangeURI = ?""", "urn:lsid:%:" + ExpProtocol.AssayDomainTypes.Result.getPrefix() + ".%", PropertyType.BOOLEAN.getTypeUri()).getObject(Long.class)); - - results.put("assayRunWithBooleanColumnCount", new SqlSelector(schema, """ - SELECT COUNT(DISTINCT DD.DomainURI) FROM - exp.PropertyDescriptor D\s - JOIN exp.PropertyDomain PD ON D.propertyId = PD.propertyid - JOIN exp.DomainDescriptor DD on PD.domainID = DD.domainId - WHERE D.propertyURI LIKE ? AND D.rangeURI = ?""", "urn:lsid:%:" + ExpProtocol.AssayDomainTypes.Run.getPrefix() + ".%", PropertyType.BOOLEAN.getTypeUri()).getObject(Long.class)); - - results.put("assay", assayMetrics); - } - - results.put("autoLinkedSampleSetCount", new SqlSelector(schema, "SELECT COUNT(*) FROM exp.materialsource WHERE autoLinkTargetContainer IS NOT NULL").getObject(Long.class)); - results.put("sampleSetCount", new SqlSelector(schema, "SELECT COUNT(*) FROM exp.materialsource").getObject(Long.class)); - - if (schema.getSqlDialect().isPostgreSQL()) // SQLServer does not support regular expression queries - { - Collection> numSampleCounts = new SqlSelector(schema, """ - SELECT totalCount, numberNameCount FROM - (SELECT cpastype, COUNT(*) AS totalCount from exp.material GROUP BY cpastype) t - JOIN - (SELECT cpastype, COUNT(*) AS numberNameCount FROM exp.material m WHERE m.name SIMILAR TO '[0-9.]*' GROUP BY cpastype) ns - ON t.cpastype = ns.cpastype""").getMapCollection(); - results.put("sampleSetWithNumberNamesCount", numSampleCounts.size()); - results.put("sampleSetWithOnlyNumberNamesCount", numSampleCounts.stream().filter( - map -> (Long) map.get("totalCount") > 0 && map.get("totalCount") == map.get("numberNameCount") - ).count()); - } - UserSchema userSchema = AuditLogService.getAuditLogSchema(User.getSearchUser(), ContainerManager.getRoot()); - FilteredTable table = (FilteredTable) userSchema.getTable(SampleTimelineAuditEvent.EVENT_TYPE); - - SQLFragment sql = new SQLFragment("SELECT COUNT(*)\n" + - " FROM (\n" + - " -- updates that are marked as lineage updates\n" + - " (SELECT DISTINCT transactionId\n" + - " FROM " + table.getRealTable().getFromSQL("").getSQL() +"\n" + - " WHERE islineageupdate = " + schema.getSqlDialect().getBooleanTRUE() + "\n" + - " AND comment = 'Sample was updated.'\n" + - " ) a1\n" + - " JOIN\n" + - " -- but have associated entries that are not lineage updates\n" + - " (SELECT DISTINCT transactionid\n" + - " FROM " + table.getRealTable().getFromSQL("").getSQL() + "\n" + - " WHERE islineageupdate = " + schema.getSqlDialect().getBooleanFALSE() + ") a2\n" + - " ON a1.transactionid = a2.transactionid\n" + - " )"); - - results.put("sampleLineageAuditDiscrepancyCount", new SqlSelector(schema, sql.getSQL()).getObject(Long.class)); - - results.put("sampleCount", new SqlSelector(schema, "SELECT COUNT(*) FROM exp.material").getObject(Long.class)); - results.put("aliquotCount", new SqlSelector(schema, "SELECT COUNT(*) FROM exp.material where aliquotedfromlsid IS NOT NULL").getObject(Long.class)); - results.put("sampleNullAmountCount", new SqlSelector(schema, "SELECT COUNT(*) FROM exp.material WHERE storedamount IS NULL").getObject(Long.class)); - results.put("sampleNegativeAmountCount", new SqlSelector(schema, "SELECT COUNT(*) FROM exp.material WHERE storedamount < 0").getObject(Long.class)); - results.put("sampleUnitsDifferCount", new SqlSelector(schema, "SELECT COUNT(*) from exp.material m JOIN exp.materialSource s ON m.materialsourceid = s.rowid WHERE m.units != s.metricunit").getObject(Long.class)); - results.put("sampleTypesWithoutUnitsCount", new SqlSelector(schema, "SELECT COUNT(*) from exp.materialSource WHERE category IS NULL AND metricunit IS NULL").getObject(Long.class)); - - results.put("duplicateSampleMaterialNameCount", new SqlSelector(schema, "SELECT COUNT(*) as duplicateCount FROM " + - "(SELECT name, cpastype FROM exp.material WHERE cpastype <> 'Material' GROUP BY name, cpastype HAVING COUNT(*) > 1) d").getObject(Long.class)); - results.put("duplicateSpecimenMaterialNameCount", new SqlSelector(schema, "SELECT COUNT(*) as duplicateCount FROM " + - "(SELECT name, cpastype FROM exp.material WHERE cpastype = 'Material' GROUP BY name, cpastype HAVING COUNT(*) > 1) d").getObject(Long.class)); - String duplicateCaseInsensitiveSampleNameCountSql = """ - SELECT COUNT(*) FROM - ( - SELECT 1 AS found - FROM exp.material - WHERE materialsourceid IS NOT NULL - GROUP BY LOWER(name), materialsourceid - HAVING COUNT(*) > 1 - ) AS duplicates - """; - String duplicateCaseInsensitiveDataNameCountSql = """ - SELECT COUNT(*) FROM - ( - SELECT 1 AS found - FROM exp.data - WHERE classid IS NOT NULL - GROUP BY LOWER(name), classid - HAVING COUNT(*) > 1 - ) AS duplicates - """; - results.put("duplicateCaseInsensitiveSampleNameCount", new SqlSelector(schema, duplicateCaseInsensitiveSampleNameCountSql).getObject(Long.class)); - results.put("duplicateCaseInsensitiveDataNameCount", new SqlSelector(schema, duplicateCaseInsensitiveDataNameCountSql).getObject(Long.class)); - - results.put("dataClassCount", new SqlSelector(schema, "SELECT COUNT(*) FROM exp.dataclass").getObject(Long.class)); - results.put("dataClassRowCount", new SqlSelector(schema, "SELECT COUNT(*) FROM exp.data WHERE classid IN (SELECT rowid FROM exp.dataclass)").getObject(Long.class)); - results.put("dataWithDataParentsCount", new SqlSelector(schema, "SELECT COUNT(DISTINCT d.sourceApplicationId) FROM exp.data d\n" + - "JOIN exp.datainput di ON di.targetapplicationid = d.sourceapplicationid").getObject(Long.class)); - if (schema.getSqlDialect().isPostgreSQL()) - { - Collection> numDataClassObjectsCounts = new SqlSelector(schema, """ - SELECT totalCount, numberNameCount FROM - (SELECT cpastype, COUNT(*) AS totalCount from exp.data GROUP BY cpastype) t - JOIN - (SELECT cpastype, COUNT(*) AS numberNameCount FROM exp.data m WHERE m.name SIMILAR TO '[0-9.]*' GROUP BY cpastype) ns - ON t.cpastype = ns.cpastype""").getMapCollection(); - results.put("dataClassWithNumberNamesCount", numDataClassObjectsCounts.size()); - results.put("dataClassWithOnlyNumberNamesCount", numDataClassObjectsCounts.stream().filter(map -> - (Long) map.get("totalCount") > 0 && map.get("totalCount") == map.get("numberNameCount")).count()); - } - - results.put("ontologyPrincipalConceptCodeCount", new SqlSelector(schema, "SELECT COUNT(*) FROM exp.propertydescriptor WHERE principalconceptcode IS NOT NULL").getObject(Long.class)); - results.put("ontologyLookupColumnCount", new SqlSelector(schema, "SELECT COUNT(*) FROM exp.propertydescriptor WHERE concepturi = ?", OntologyService.conceptCodeConceptURI).getObject(Long.class)); - results.put("ontologyConceptSubtreeCount", new SqlSelector(schema, "SELECT COUNT(*) FROM exp.propertydescriptor WHERE conceptsubtree IS NOT NULL").getObject(Long.class)); - results.put("ontologyConceptImportColumnCount", new SqlSelector(schema, "SELECT COUNT(*) FROM exp.propertydescriptor WHERE conceptimportcolumn IS NOT NULL").getObject(Long.class)); - results.put("ontologyConceptLabelColumnCount", new SqlSelector(schema, "SELECT COUNT(*) FROM exp.propertydescriptor WHERE conceptlabelcolumn IS NOT NULL").getObject(Long.class)); - - results.put("scannableColumnCount", new SqlSelector(schema, "SELECT COUNT(*) FROM exp.propertydescriptor WHERE scannable = ?", true).getObject(Long.class)); - results.put("uniqueIdColumnCount", new SqlSelector(schema, "SELECT COUNT(*) FROM exp.propertydescriptor WHERE concepturi = ?", STORAGE_UNIQUE_ID_CONCEPT_URI).getObject(Long.class)); - results.put("sampleTypeWithUniqueIdCount", new SqlSelector(schema, """ - SELECT COUNT(DISTINCT DD.DomainURI) FROM - exp.PropertyDescriptor D\s - JOIN exp.PropertyDomain PD ON D.propertyId = PD.propertyid - JOIN exp.DomainDescriptor DD on PD.domainID = DD.domainId - WHERE D.conceptURI = ?""", STORAGE_UNIQUE_ID_CONCEPT_URI).getObject(Long.class)); - - results.put("fileColumnCount", new SqlSelector(schema, "SELECT COUNT(*) FROM exp.propertydescriptor WHERE rangeURI = ?", PropertyType.FILE_LINK.getTypeUri()).getObject(Long.class)); - results.put("sampleTypeWithFileColumnCount", new SqlSelector(schema, """ - SELECT COUNT(DISTINCT DD.DomainURI) FROM - exp.PropertyDescriptor D\s - JOIN exp.PropertyDomain PD ON D.propertyId = PD.propertyid - JOIN exp.DomainDescriptor DD on PD.domainID = DD.domainId - WHERE DD.storageSchemaName = ? AND D.rangeURI = ?""", SampleTypeDomainKind.PROVISIONED_SCHEMA_NAME, PropertyType.FILE_LINK.getTypeUri()).getObject(Long.class)); - results.put("sampleTypeWithBooleanColumnCount", new SqlSelector(schema, """ - SELECT COUNT(DISTINCT DD.DomainURI) FROM - exp.PropertyDescriptor D\s - JOIN exp.PropertyDomain PD ON D.propertyId = PD.propertyid - JOIN exp.DomainDescriptor DD on PD.domainID = DD.domainId - WHERE DD.storageSchemaName = ? AND D.rangeURI = ?""", SampleTypeDomainKind.PROVISIONED_SCHEMA_NAME, PropertyType.BOOLEAN.getTypeUri()).getObject(Long.class)); - - results.put("sampleTypeAliquotSpecificField", new SqlSelector(schema, """ - SELECT COUNT(DISTINCT D.PropertyURI) FROM - exp.PropertyDescriptor D\s - JOIN exp.PropertyDomain PD ON D.propertyId = PD.propertyid - JOIN exp.DomainDescriptor DD on PD.domainID = DD.domainId - WHERE DD.storageSchemaName = ? AND D.derivationDataScope = ?""", SampleTypeDomainKind.PROVISIONED_SCHEMA_NAME, ExpSchema.DerivationDataScopeType.ChildOnly.name()).getObject(Long.class)); - results.put("sampleTypeParentOnlyField", new SqlSelector(schema, """ - SELECT COUNT(DISTINCT D.PropertyURI) FROM - exp.PropertyDescriptor D\s - JOIN exp.PropertyDomain PD ON D.propertyId = PD.propertyid - JOIN exp.DomainDescriptor DD on PD.domainID = DD.domainId - WHERE DD.storageSchemaName = ? AND (D.derivationDataScope = ? OR D.derivationDataScope IS NULL)""", SampleTypeDomainKind.PROVISIONED_SCHEMA_NAME, ExpSchema.DerivationDataScopeType.ParentOnly.name()).getObject(Long.class)); - results.put("sampleTypeParentAndAliquotField", new SqlSelector(schema, """ - SELECT COUNT(DISTINCT D.PropertyURI) FROM - exp.PropertyDescriptor D\s - JOIN exp.PropertyDomain PD ON D.propertyId = PD.propertyid - JOIN exp.DomainDescriptor DD on PD.domainID = DD.domainId - WHERE DD.storageSchemaName = ? AND D.derivationDataScope = ?""", SampleTypeDomainKind.PROVISIONED_SCHEMA_NAME, ExpSchema.DerivationDataScopeType.All.name()).getObject(Long.class)); - - results.put("attachmentColumnCount", new SqlSelector(schema, "SELECT COUNT(*) FROM exp.propertydescriptor WHERE rangeURI = ?", PropertyType.ATTACHMENT.getTypeUri()).getObject(Long.class)); - results.put("dataClassWithAttachmentColumnCount", new SqlSelector(schema, """ - SELECT COUNT(DISTINCT DD.DomainURI) FROM - exp.PropertyDescriptor D\s - JOIN exp.PropertyDomain PD ON D.propertyId = PD.propertyid - JOIN exp.DomainDescriptor DD on PD.domainID = DD.domainId - WHERE DD.storageSchemaName = ? AND D.rangeURI = ?""", DataClassDomainKind.PROVISIONED_SCHEMA_NAME, PropertyType.ATTACHMENT.getTypeUri()).getObject(Long.class)); - results.put("dataClassWithBooleanColumnCount", new SqlSelector(schema, """ - SELECT COUNT(DISTINCT DD.DomainURI) FROM - exp.PropertyDescriptor D\s - JOIN exp.PropertyDomain PD ON D.propertyId = PD.propertyid - JOIN exp.DomainDescriptor DD on PD.domainID = DD.domainId - WHERE DD.storageSchemaName = ? AND D.rangeURI = ?""", DataClassDomainKind.PROVISIONED_SCHEMA_NAME, PropertyType.BOOLEAN.getTypeUri()).getObject(Long.class)); - - results.put("textChoiceColumnCount", new SqlSelector(schema, "SELECT COUNT(*) FROM exp.propertydescriptor WHERE concepturi = ?", TEXT_CHOICE_CONCEPT_URI).getObject(Long.class)); - - results.put("domainsWithDateTimeColumnCount", new SqlSelector(schema, """ - SELECT COUNT(DISTINCT DD.DomainURI) FROM - exp.PropertyDescriptor D\s - JOIN exp.PropertyDomain PD ON D.propertyId = PD.propertyid - JOIN exp.DomainDescriptor DD on PD.domainID = DD.domainId - WHERE D.rangeURI = ?""", PropertyType.DATE_TIME.getTypeUri()).getObject(Long.class)); - - results.put("domainsWithDateColumnCount", new SqlSelector(schema, """ - SELECT COUNT(DISTINCT DD.DomainURI) FROM - exp.PropertyDescriptor D\s - JOIN exp.PropertyDomain PD ON D.propertyId = PD.propertyid - JOIN exp.DomainDescriptor DD on PD.domainID = DD.domainId - WHERE D.rangeURI = ?""", PropertyType.DATE.getTypeUri()).getObject(Long.class)); - - results.put("domainsWithTimeColumnCount", new SqlSelector(schema, """ - SELECT COUNT(DISTINCT DD.DomainURI) FROM - exp.PropertyDescriptor D\s - JOIN exp.PropertyDomain PD ON D.propertyId = PD.propertyid - JOIN exp.DomainDescriptor DD on PD.domainID = DD.domainId - WHERE D.rangeURI = ?""", PropertyType.TIME.getTypeUri()).getObject(Long.class)); - - results.put("maxObjectObjectId", new SqlSelector(schema, "SELECT MAX(ObjectId) FROM exp.Object").getObject(Long.class)); - results.put("maxMaterialRowId", new SqlSelector(schema, "SELECT MAX(RowId) FROM exp.Material").getObject(Long.class)); - - results.putAll(ExperimentService.get().getDomainMetrics()); - - return results; - }); - } - - ExperimentMigrationSchemaHandler handler = new ExperimentMigrationSchemaHandler(); - DatabaseMigrationService.get().registerSchemaHandler(handler); - DatabaseMigrationService.get().registerTableHandler(new MigrationTableHandler() - { - @Override - public TableInfo getTableInfo() - { - return DbSchema.get("premium", DbSchemaType.Bare).getTable("Exclusions"); - } - - @Override - public void adjustFilter(TableInfo sourceTable, SimpleFilter filter, Set containers) - { - // Include experiment runs that were copied - FilterClause includedClause = handler.getIncludedRowIdClause(sourceTable, FieldKey.fromParts("RunId")); - if (includedClause != null) - filter.addClause(includedClause); - } - }); - DatabaseMigrationService.get().registerTableHandler(new MigrationTableHandler() - { - @Override - public TableInfo getTableInfo() - { - return DbSchema.get("premium", DbSchemaType.Bare).getTable("ExclusionMaps"); - } - - @Override - public void adjustFilter(TableInfo sourceTable, SimpleFilter filter, Set containers) - { - // Include experiment runs that were copied - FilterClause includedClause = handler.getIncludedRowIdClause(sourceTable, FieldKey.fromParts("ExclusionId", "RunId")); - if (includedClause != null) - filter.addClause(includedClause); - } - }); - DatabaseMigrationService.get().registerTableHandler(new MigrationTableHandler() - { - @Override - public TableInfo getTableInfo() - { - return DbSchema.get("assayrequest", DbSchemaType.Bare).getTable("RequestRunsJunction"); - } - - @Override - public void adjustFilter(TableInfo sourceTable, SimpleFilter filter, Set containers) - { - // Include experiment runs that were copied - FilterClause includedClause = handler.getIncludedRowIdClause(sourceTable, FieldKey.fromParts("RunId")); - if (includedClause != null) - filter.addClause(includedClause); - } - }); - DatabaseMigrationService.get().registerSchemaHandler(new SampleTypeMigrationSchemaHandler()); - DataClassMigrationSchemaHandler dcHandler = new DataClassMigrationSchemaHandler(); - DatabaseMigrationService.get().registerSchemaHandler(dcHandler); - ExperimentDeleteService.setInstance(dcHandler); - } - - @Override - @NotNull - public Collection getSummary(Container c) - { - Collection list = new LinkedList<>(); - int runGroupCount = ExperimentService.get().getExperiments(c, null, false, true).size(); - if (runGroupCount > 0) - list.add(StringUtilsLabKey.pluralize(runGroupCount, "Run Group")); - - User user = HttpView.currentContext().getUser(); - - Set runTypes = ExperimentService.get().getExperimentRunTypes(c); - for (ExperimentRunType runType : runTypes) - { - if (runType == ExperimentRunType.ALL_RUNS_TYPE) - continue; - - long runCount = runType.getRunCount(user, c); - if (runCount > 0) - list.add(runCount + " runs of type " + runType.getDescription()); - } - - int dataClassCount = ExperimentService.get().getDataClasses(c, null, false).size(); - if (dataClassCount > 0) - list.add(dataClassCount + " Data Class" + (dataClassCount > 1 ? "es" : "")); - - int sampleTypeCount = SampleTypeService.get().getSampleTypes(c, false).size(); - if (sampleTypeCount > 0) - list.add(sampleTypeCount + " Sample Type" + (sampleTypeCount > 1 ? "s" : "")); - - return list; - } - - @Override - public @NotNull ArrayList getDetailedSummary(Container c, User user) - { - ArrayList summaries = new ArrayList<>(); - - // Assay types - long assayTypeCount = AssayService.get().getAssayProtocols(c).stream().filter(p -> p.getContainer().equals(c)).count(); - if (assayTypeCount > 0) - summaries.add(new Summary(assayTypeCount, "Assay Type")); - - // Run count - int runGroupCount = ExperimentService.get().getExperiments(c, user, false, true).size(); - if (runGroupCount > 0) - summaries.add(new Summary(runGroupCount, "Assay run")); - - // Number of Data Classes - List dataClasses = ExperimentService.get().getDataClasses(c, user, false); - int dataClassCount = dataClasses.size(); - if (dataClassCount > 0) - summaries.add(new Summary(dataClassCount, "Data Class")); - - ExpSchema expSchema = new ExpSchema(user, c); - - // Individual Data Class row counts - { - // The table-level container filter is set to ensure data class types are included - // that may not be defined in the target container but may have rows of data in the target container - TableInfo table = ExpSchema.TableType.DataClasses.createTable(expSchema, null, ContainerFilter.Type.CurrentPlusProjectAndShared.create(c, user)); - - // Issue 47919: The "DataCount" column is filtered to only count data in the target container - if (table instanceof ExpDataClassTableImpl tableImpl) - tableImpl.setDataCountContainerFilter(ContainerFilter.Type.Current.create(c, user)); - - Set columns = new LinkedHashSet<>(); - columns.add(ExpDataClassTable.Column.Name.name()); - columns.add(ExpDataClassTable.Column.DataCount.name()); - - Map results = new TableSelector(table, columns).getValueMap(String.class); - for (var entry : results.entrySet()) - { - long count = entry.getValue().longValue(); - if (count > 0) - summaries.add(new Summary(count, entry.getKey())); - } - } - - // Sample Types - int sampleTypeCount = SampleTypeService.get().getSampleTypes(c, false).size(); - if (sampleTypeCount > 0) - summaries.add(new Summary(sampleTypeCount, "Sample Type")); - - // Individual Sample Type row counts - { - // The table-level container filter is set to ensure data class types are included - // that may not be defined in the target container but may have rows of data in the target container - TableInfo table = ExpSchema.TableType.SampleSets.createTable(expSchema, null, ContainerFilter.Type.CurrentPlusProjectAndShared.create(c, user)); - - // Issue 51557: The "SampleCount" column is filtered to only count data in the target container - if (table instanceof ExpSampleTypeTableImpl tableImpl) - tableImpl.setSampleCountContainerFilter(ContainerFilter.Type.Current.create(c, user)); - - Set columns = new LinkedHashSet<>(); - columns.add(ExpSampleTypeTable.Column.Name.name()); - columns.add(ExpSampleTypeTable.Column.SampleCount.name()); - - Map results = new TableSelector(table, columns).getValueMap(String.class); - for (var entry : results.entrySet()) - { - long count = entry.getValue().longValue(); - if (count > 0) - { - String name = entry.getKey(); - Summary s = name.equals("MixtureBatches") - ? new Summary(count, "Batch") - : new Summary(count, name); - summaries.add(s); - } - } - } - - return summaries; - } - - @Override - public @NotNull Set> getIntegrationTests() - { - return Set.of( - DomainImpl.TestCase.class, - DomainPropertyImpl.TestCase.class, - ExpDataTableImpl.TestCase.class, - ExperimentServiceImpl.AuditDomainUriTest.class, - ExperimentServiceImpl.LineageQueryTestCase.class, - ExperimentServiceImpl.ParseInputOutputAliasTestCase.class, - ExperimentServiceImpl.TestCase.class, - ExperimentStressTest.class, - LineagePerfTest.class, - LineageTest.class, - OntologyManager.TestCase.class, - PropertyServiceImpl.TestCase.class, - SampleTypeServiceImpl.TestCase.class, - StorageNameGenerator.TestCase.class, - StorageProvisionerImpl.TestCase.class, - UniqueValueCounterTestCase.class, - XarTestPipelineJob.TestCase.class - ); - } - - @Override - public @NotNull Collection>> getIntegrationTestFactories() - { - List>> list = new ArrayList<>(super.getIntegrationTestFactories()); - list.add(new JspTestCase("/org/labkey/experiment/api/ExpDataClassDataTestCase.jsp")); - list.add(new JspTestCase("/org/labkey/experiment/api/ExpSampleTypeTestCase.jsp")); - return list; - } - - @Override - public @NotNull Set> getUnitTests() - { - return Set.of( - GraphAlgorithms.TestCase.class, - LSIDRelativizer.TestCase.class, - Lsid.TestCase.class, - LsidUtils.TestCase.class, - PropertyController.TestCase.class, - Quantity.TestCase.class, - Unit.TestCase.class - ); - } - - @Override - @NotNull - public Collection getSchemaNames() - { - return List.of( - ExpSchema.SCHEMA_NAME, - DataClassDomainKind.PROVISIONED_SCHEMA_NAME, - SampleTypeDomainKind.PROVISIONED_SCHEMA_NAME - ); - } - - @NotNull - @Override - public Collection getProvisionedSchemaNames() - { - return PageFlowUtil.set(DataClassDomainKind.PROVISIONED_SCHEMA_NAME, SampleTypeDomainKind.PROVISIONED_SCHEMA_NAME); - } -} +/* + * Copyright (c) 2008-2019 LabKey Corporation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.labkey.experiment; + +import org.apache.commons.lang3.math.NumberUtils; +import org.jetbrains.annotations.NotNull; +import org.jetbrains.annotations.Nullable; +import org.labkey.api.admin.FolderSerializationRegistry; +import org.labkey.api.assay.AssayProvider; +import org.labkey.api.assay.AssayService; +import org.labkey.api.attachments.AttachmentService; +import org.labkey.api.audit.AuditLogService; +import org.labkey.api.audit.SampleTimelineAuditEvent; +import org.labkey.api.collections.LongHashMap; +import org.labkey.api.data.Container; +import org.labkey.api.data.ContainerFilter; +import org.labkey.api.data.ContainerManager; +import org.labkey.api.data.CoreSchema; +import org.labkey.api.data.DbSchema; +import org.labkey.api.data.DbSchemaType; +import org.labkey.api.data.JdbcType; +import org.labkey.api.data.NameGenerator; +import org.labkey.api.data.SQLFragment; +import org.labkey.api.data.SimpleFilter; +import org.labkey.api.data.SimpleFilter.FilterClause; +import org.labkey.api.data.SqlSelector; +import org.labkey.api.data.TableInfo; +import org.labkey.api.data.TableSelector; +import org.labkey.api.data.UpgradeCode; +import org.labkey.api.defaults.DefaultValueService; +import org.labkey.api.exp.ExperimentException; +import org.labkey.api.exp.ExperimentRunType; +import org.labkey.api.exp.Lsid; +import org.labkey.api.exp.OntologyManager; +import org.labkey.api.exp.PropertyType; +import org.labkey.api.exp.api.DefaultExperimentDataHandler; +import org.labkey.api.exp.api.ExpData; +import org.labkey.api.exp.api.ExpDataClass; +import org.labkey.api.exp.api.ExpLineageService; +import org.labkey.api.exp.api.ExpMaterial; +import org.labkey.api.exp.api.ExpProtocol; +import org.labkey.api.exp.api.ExpProtocolAttachmentType; +import org.labkey.api.exp.api.ExpRunAttachmentType; +import org.labkey.api.exp.api.ExpSampleType; +import org.labkey.api.exp.api.ExperimentJSONConverter; +import org.labkey.api.exp.api.ExperimentService; +import org.labkey.api.exp.api.FilterProtocolInputCriteria; +import org.labkey.api.exp.api.SampleTypeDomainKind; +import org.labkey.api.exp.api.SampleTypeService; +import org.labkey.api.exp.api.StorageProvisioner; +import org.labkey.api.exp.property.DomainAuditProvider; +import org.labkey.api.exp.property.DomainPropertyAuditProvider; +import org.labkey.api.exp.property.ExperimentProperty; +import org.labkey.api.exp.property.PropertyService; +import org.labkey.api.exp.property.SystemProperty; +import org.labkey.api.exp.query.ExpDataClassTable; +import org.labkey.api.exp.query.ExpSampleTypeTable; +import org.labkey.api.exp.query.ExpSchema; +import org.labkey.api.exp.query.SamplesSchema; +import org.labkey.api.exp.xar.LSIDRelativizer; +import org.labkey.api.exp.xar.LsidUtils; +import org.labkey.api.files.FileContentService; +import org.labkey.api.files.TableUpdaterFileListener; +import org.labkey.api.migration.DatabaseMigrationService; +import org.labkey.api.migration.ExperimentDeleteService; +import org.labkey.api.migration.MigrationTableHandler; +import org.labkey.api.module.ModuleContext; +import org.labkey.api.module.ModuleLoader; +import org.labkey.api.module.SpringModule; +import org.labkey.api.module.Summary; +import org.labkey.api.ontology.OntologyService; +import org.labkey.api.ontology.Quantity; +import org.labkey.api.ontology.Unit; +import org.labkey.api.pipeline.PipelineService; +import org.labkey.api.query.FieldKey; +import org.labkey.api.query.FilteredTable; +import org.labkey.api.query.QueryService; +import org.labkey.api.query.UserSchema; +import org.labkey.api.search.SearchService; +import org.labkey.api.security.User; +import org.labkey.api.security.roles.RoleManager; +import org.labkey.api.settings.AppProps; +import org.labkey.api.settings.OptionalFeatureService; +import org.labkey.api.usageMetrics.UsageMetricsService; +import org.labkey.api.util.GUID; +import org.labkey.api.util.JspTestCase; +import org.labkey.api.util.PageFlowUtil; +import org.labkey.api.util.StringUtilsLabKey; +import org.labkey.api.util.SystemMaintenance; +import org.labkey.api.view.AlwaysAvailableWebPartFactory; +import org.labkey.api.view.BaseWebPartFactory; +import org.labkey.api.view.HttpView; +import org.labkey.api.view.JspView; +import org.labkey.api.view.Portal; +import org.labkey.api.view.ViewContext; +import org.labkey.api.view.WebPartFactory; +import org.labkey.api.view.WebPartView; +import org.labkey.api.view.template.WarningService; +import org.labkey.api.vocabulary.security.DesignVocabularyPermission; +import org.labkey.api.webdav.WebdavResource; +import org.labkey.api.webdav.WebdavService; +import org.labkey.experiment.api.DataClassDomainKind; +import org.labkey.experiment.api.ExpDataClassImpl; +import org.labkey.experiment.api.ExpDataClassTableImpl; +import org.labkey.experiment.api.ExpDataClassType; +import org.labkey.experiment.api.ExpDataImpl; +import org.labkey.experiment.api.ExpDataTableImpl; +import org.labkey.experiment.api.ExpMaterialImpl; +import org.labkey.experiment.api.ExpProtocolImpl; +import org.labkey.experiment.api.ExpSampleTypeImpl; +import org.labkey.experiment.api.ExpSampleTypeTableImpl; +import org.labkey.experiment.api.ExperimentServiceImpl; +import org.labkey.experiment.api.ExperimentStressTest; +import org.labkey.experiment.api.GraphAlgorithms; +import org.labkey.experiment.api.LineageTest; +import org.labkey.experiment.api.LogDataType; +import org.labkey.experiment.api.Protocol; +import org.labkey.experiment.api.SampleTypeServiceImpl; +import org.labkey.experiment.api.UniqueValueCounterTestCase; +import org.labkey.experiment.api.VocabularyDomainKind; +import org.labkey.experiment.api.data.ChildOfCompareType; +import org.labkey.experiment.api.data.ChildOfMethod; +import org.labkey.experiment.api.data.LineageCompareType; +import org.labkey.experiment.api.data.ParentOfCompareType; +import org.labkey.experiment.api.data.ParentOfMethod; +import org.labkey.experiment.api.property.DomainImpl; +import org.labkey.experiment.api.property.DomainPropertyImpl; +import org.labkey.experiment.api.property.LengthValidator; +import org.labkey.experiment.api.property.LookupValidator; +import org.labkey.experiment.api.property.PropertyServiceImpl; +import org.labkey.experiment.api.property.RangeValidator; +import org.labkey.experiment.api.property.RegExValidator; +import org.labkey.experiment.api.property.StorageNameGenerator; +import org.labkey.experiment.api.property.StorageProvisionerImpl; +import org.labkey.experiment.api.property.TextChoiceValidator; +import org.labkey.experiment.controllers.exp.ExperimentController; +import org.labkey.experiment.controllers.property.PropertyController; +import org.labkey.experiment.defaults.DefaultValueServiceImpl; +import org.labkey.experiment.lineage.ExpLineageServiceImpl; +import org.labkey.experiment.lineage.LineagePerfTest; +import org.labkey.experiment.pipeline.ExperimentPipelineProvider; +import org.labkey.experiment.pipeline.XarTestPipelineJob; +import org.labkey.experiment.samples.DataClassFolderImporter; +import org.labkey.experiment.samples.DataClassFolderWriter; +import org.labkey.experiment.samples.SampleStatusFolderImporter; +import org.labkey.experiment.samples.SampleTimelineAuditProvider; +import org.labkey.experiment.samples.SampleTypeFolderImporter; +import org.labkey.experiment.samples.SampleTypeFolderWriter; +import org.labkey.experiment.security.DataClassDesignerRole; +import org.labkey.experiment.security.SampleTypeDesignerRole; +import org.labkey.experiment.types.TypesController; +import org.labkey.experiment.xar.FolderXarImporterFactory; +import org.labkey.experiment.xar.FolderXarWriterFactory; + +import java.util.ArrayList; +import java.util.Collection; +import java.util.Collections; +import java.util.HashMap; +import java.util.HashSet; +import java.util.LinkedHashSet; +import java.util.LinkedList; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.function.Supplier; +import java.util.stream.Collectors; + +import static org.labkey.api.data.ColumnRenderPropertiesImpl.STORAGE_UNIQUE_ID_CONCEPT_URI; +import static org.labkey.api.data.ColumnRenderPropertiesImpl.TEXT_CHOICE_CONCEPT_URI; +import static org.labkey.api.exp.api.ExperimentService.MODULE_NAME; +import static org.labkey.api.exp.query.ExpSchema.SAMPLE_FILES_TABLE; + +public class ExperimentModule extends SpringModule +{ + private static final String SAMPLE_TYPE_WEB_PART_NAME = "Sample Types"; + private static final String PROTOCOL_WEB_PART_NAME = "Protocols"; + + public static final String AMOUNT_AND_UNIT_UPGRADE_PROP = "AmountAndUnitAudit"; + public static final String TRANSACTION_ID_PROP = "AuditTransactionId"; + public static final String AUDIT_COUNT_PROP = "AuditRecordCount"; + public static final String EXPERIMENT_RUN_WEB_PART_NAME = "Experiment Runs"; + + @Override + public String getName() + { + return MODULE_NAME; + } + + @Override + public Double getSchemaVersion() + { + return 25.013; + } + + @Nullable + @Override + public UpgradeCode getUpgradeCode() + { + return new ExperimentUpgradeCode(); + } + + @Override + protected void init() + { + addController("experiment", ExperimentController.class); + addController("experiment-types", TypesController.class); + addController("property", PropertyController.class); + ExperimentService.setInstance(new ExperimentServiceImpl()); + SampleTypeService.setInstance(new SampleTypeServiceImpl()); + DefaultValueService.setInstance(new DefaultValueServiceImpl()); + StorageProvisioner.setInstance(StorageProvisionerImpl.get()); + ExpLineageService.setInstance(new ExpLineageServiceImpl()); + + PropertyServiceImpl propertyServiceImpl = new PropertyServiceImpl(); + PropertyService.setInstance(propertyServiceImpl); + UsageMetricsService.get().registerUsageMetrics(getName(), propertyServiceImpl); + + UsageMetricsService.get().registerUsageMetrics(getName(), FileLinkMetricsProvider.getInstance()); + + ExperimentProperty.register(); + SamplesSchema.register(this); + ExpSchema.register(this); + + PropertyService.get().registerDomainKind(new SampleTypeDomainKind()); + PropertyService.get().registerDomainKind(new DataClassDomainKind()); + PropertyService.get().registerDomainKind(new VocabularyDomainKind()); + + QueryService.get().addCompareType(new ChildOfCompareType()); + QueryService.get().addCompareType(new ParentOfCompareType()); + QueryService.get().addCompareType(new LineageCompareType()); + QueryService.get().registerMethod(ChildOfMethod.NAME, new ChildOfMethod(), JdbcType.BOOLEAN, 2, 3); + QueryService.get().registerMethod(ParentOfMethod.NAME, new ParentOfMethod(), JdbcType.BOOLEAN, 2, 3); + QueryService.get().addQueryListener(new ExperimentQueryChangeListener()); + QueryService.get().addQueryListener(new PropertyQueryChangeListener()); + + PropertyService.get().registerValidatorKind(new RegExValidator()); + PropertyService.get().registerValidatorKind(new RangeValidator()); + PropertyService.get().registerValidatorKind(new LookupValidator()); + PropertyService.get().registerValidatorKind(new LengthValidator()); + PropertyService.get().registerValidatorKind(new TextChoiceValidator()); + + ExperimentService.get().registerExperimentDataHandler(new DefaultExperimentDataHandler()); + ExperimentService.get().registerProtocolInputCriteria(new FilterProtocolInputCriteria.Factory()); + ExperimentService.get().registerNameExpressionType("sampletype", "exp", "MaterialSource", "nameexpression"); + ExperimentService.get().registerNameExpressionType("aliquots", "exp", "MaterialSource", "aliquotnameexpression"); + ExperimentService.get().registerNameExpressionType("dataclass", "exp", "DataClass", "nameexpression"); + + OptionalFeatureService.get().addExperimentalFeatureFlag(AppProps.EXPERIMENTAL_RESOLVE_PROPERTY_URI_COLUMNS, "Resolve property URIs as columns on experiment tables", + "If a column is not found on an experiment table, attempt to resolve the column name as a Property URI and add it as a property column", false); + if (CoreSchema.getInstance().getSqlDialect().isSqlServer()) + { + OptionalFeatureService.get().addExperimentalFeatureFlag(NameGenerator.EXPERIMENTAL_WITH_COUNTER, "Use strict incremental withCounter and rootSampleCount expression", + "When withCounter or rootSampleCount is used in name expression, make sure the count increments one-by-one and does not jump.", true); + } + else + { + OptionalFeatureService.get().addExperimentalFeatureFlag(SAMPLE_FILES_TABLE, "Manage Stale Sample Files", + "Allow 'Manage Sample Files' to view sample files that are no longer referenced by samples", false); + + OptionalFeatureService.get().addExperimentalFeatureFlag(NameGenerator.EXPERIMENTAL_ALLOW_GAP_COUNTER, "Allow gap with withCounter and rootSampleCount expression", + "Check this option if gaps in the count generated by withCounter or rootSampleCount name expression are allowed.", true); + } + OptionalFeatureService.get().addExperimentalFeatureFlag(AppProps.QUANTITY_COLUMN_SUFFIX_TESTING, "Quantity column suffix testing", + "If a column name contains a \"__\" suffix, this feature allows for testing it as a Quantity display column", false); + + RoleManager.registerPermission(new DesignVocabularyPermission(), true); + RoleManager.registerRole(new SampleTypeDesignerRole()); + RoleManager.registerRole(new DataClassDesignerRole()); + + AttachmentService.get().registerAttachmentType(ExpRunAttachmentType.get()); + AttachmentService.get().registerAttachmentType(ExpProtocolAttachmentType.get()); + + WebdavService.get().addExpDataProvider((path, container) -> ExperimentService.get().getAllExpDataByURL(path, container)); + ExperimentService.get().registerObjectReferencer(ExperimentServiceImpl.get()); + + addModuleProperty(new LineageMaximumDepthModuleProperty(this)); + WarningService.get().register(new ExperimentWarningProvider()); + } + + @Override + public boolean hasScripts() + { + return true; + } + + @Override + @NotNull + protected Collection createWebPartFactories() + { + List result = new ArrayList<>(); + + BaseWebPartFactory runGroupsFactory = new BaseWebPartFactory(RunGroupWebPart.WEB_PART_NAME, WebPartFactory.LOCATION_BODY, WebPartFactory.LOCATION_RIGHT) + { + @Override + public WebPartView getWebPartView(@NotNull ViewContext portalCtx, @NotNull Portal.WebPart webPart) + { + return new RunGroupWebPart(portalCtx, WebPartFactory.LOCATION_RIGHT.equalsIgnoreCase(webPart.getLocation()), webPart); + } + }; + runGroupsFactory.addLegacyNames("Experiments", "Experiment", "Experiment Navigator", "Narrow Experiments"); + result.add(runGroupsFactory); + + BaseWebPartFactory runTypesFactory = new BaseWebPartFactory(RunTypeWebPart.WEB_PART_NAME, WebPartFactory.LOCATION_BODY, WebPartFactory.LOCATION_RIGHT) + { + @Override + public WebPartView getWebPartView(@NotNull ViewContext portalCtx, @NotNull Portal.WebPart webPart) + { + return new RunTypeWebPart(); + } + }; + result.add(runTypesFactory); + + result.add(new ExperimentRunWebPartFactory()); + BaseWebPartFactory sampleTypeFactory = new BaseWebPartFactory(SAMPLE_TYPE_WEB_PART_NAME, WebPartFactory.LOCATION_BODY, WebPartFactory.LOCATION_RIGHT) + { + @Override + public WebPartView getWebPartView(@NotNull ViewContext portalCtx, @NotNull Portal.WebPart webPart) + { + return new SampleTypeWebPart(WebPartFactory.LOCATION_RIGHT.equalsIgnoreCase(webPart.getLocation()), portalCtx); + } + }; + sampleTypeFactory.addLegacyNames("Narrow Sample Sets", "Sample Sets"); + result.add(sampleTypeFactory); + result.add(new AlwaysAvailableWebPartFactory("Samples Menu", false, false, WebPartFactory.LOCATION_MENUBAR) { + @Override + public WebPartView getWebPartView(@NotNull ViewContext portalCtx, @NotNull Portal.WebPart webPart) + { + WebPartView view = new JspView<>("/org/labkey/experiment/samplesAndAnalytes.jsp", webPart); + view.setTitle("Samples"); + return view; + } + }); + + result.add(new AlwaysAvailableWebPartFactory("Data Classes", false, false, WebPartFactory.LOCATION_BODY, WebPartFactory.LOCATION_RIGHT) { + @Override + public WebPartView getWebPartView(@NotNull ViewContext portalCtx, @NotNull Portal.WebPart webPart) + { + return new DataClassWebPart(WebPartFactory.LOCATION_RIGHT.equalsIgnoreCase(webPart.getLocation()), portalCtx, webPart); + } + }); + + BaseWebPartFactory narrowProtocolFactory = new BaseWebPartFactory(PROTOCOL_WEB_PART_NAME, WebPartFactory.LOCATION_RIGHT) + { + @Override + public WebPartView getWebPartView(@NotNull ViewContext portalCtx, @NotNull Portal.WebPart webPart) + { + return new ProtocolWebPart(WebPartFactory.LOCATION_RIGHT.equalsIgnoreCase(webPart.getLocation()), portalCtx); + } + }; + narrowProtocolFactory.addLegacyNames("Narrow Protocols"); + result.add(narrowProtocolFactory); + + return result; + } + + private void addDataResourceResolver(String categoryName) + { + SearchService.get().addResourceResolver(categoryName, new SearchService.ResourceResolver() + { + @Override + public WebdavResource resolve(@NotNull String resourceIdentifier) + { + ExpDataImpl data = ExpDataImpl.fromDocumentId(resourceIdentifier); + if (data == null) + return null; + + return data.createIndexDocument(null); + } + + @Override + public Map getCustomSearchJson(User user, @NotNull String resourceIdentifier) + { + ExpDataImpl data = ExpDataImpl.fromDocumentId(resourceIdentifier); + if (data == null) + return null; + + return ExperimentJSONConverter.serializeData(data, user, ExperimentJSONConverter.DEFAULT_SETTINGS).toMap(); + } + + @Override + public Map> getCustomSearchJsonMap(User user, @NotNull Collection resourceIdentifiers) + { + Map idDataMap = ExpDataImpl.fromDocumentIds(resourceIdentifiers); + if (idDataMap == null) + return null; + + Map> searchJsonMap = new HashMap<>(); + for (String resourceIdentifier : idDataMap.keySet()) + searchJsonMap.put(resourceIdentifier, ExperimentJSONConverter.serializeData(idDataMap.get(resourceIdentifier), user, ExperimentJSONConverter.DEFAULT_SETTINGS).toMap()); + return searchJsonMap; + } + }); + } + + private void addDataClassResourceResolver(String categoryName) + { + SearchService.get().addResourceResolver(categoryName, new SearchService.ResourceResolver(){ + @Override + public Map getCustomSearchJson(User user, @NotNull String resourceIdentifier) + { + int rowId = NumberUtils.toInt(resourceIdentifier.replace(categoryName + ":", "")); + if (rowId == 0) + return null; + + ExpDataClass dataClass = ExperimentService.get().getDataClass(rowId); + if (dataClass == null) + return null; + + Map properties = ExperimentJSONConverter.serializeExpObject(dataClass, null, ExperimentJSONConverter.DEFAULT_SETTINGS, user).toMap(); + + //Need to map to proper Icon + properties.put("type", "dataClass" + (dataClass.getCategory() != null ? ":" + dataClass.getCategory() : "")); + + return properties; + } + }); + } + + private void addSampleTypeResourceResolver(String categoryName) + { + SearchService.get().addResourceResolver(categoryName, new SearchService.ResourceResolver(){ + @Override + public Map getCustomSearchJson(User user, @NotNull String resourceIdentifier) + { + int rowId = NumberUtils.toInt(resourceIdentifier.replace(categoryName + ":", "")); + if (rowId == 0) + return null; + + ExpSampleType sampleType = SampleTypeService.get().getSampleType(rowId); + if (sampleType == null) + return null; + + Map properties = ExperimentJSONConverter.serializeExpObject(sampleType, null, ExperimentJSONConverter.DEFAULT_SETTINGS, user).toMap(); + + //Need to map to proper Icon + properties.put("type", "sampleSet"); + + return properties; + } + }); + } + + private void addSampleResourceResolver(String categoryName) + { + SearchService.get().addResourceResolver(categoryName, new SearchService.ResourceResolver(){ + @Override + public Map getCustomSearchJson(User user, @NotNull String resourceIdentifier) + { + int rowId = NumberUtils.toInt(resourceIdentifier.replace(categoryName + ":", "")); + if (rowId == 0) + return null; + + ExpMaterial material = ExperimentService.get().getExpMaterial(rowId); + if (material == null) + return null; + + return ExperimentJSONConverter.serializeMaterial(material, user, ExperimentJSONConverter.DEFAULT_SETTINGS).toMap(); + } + + @Override + public Map> getCustomSearchJsonMap(User user, @NotNull Collection resourceIdentifiers) + { + Set rowIds = new HashSet<>(); + Map rowIdIdentifierMap = new LongHashMap<>(); + for (String resourceIdentifier : resourceIdentifiers) + { + long rowId = NumberUtils.toLong(resourceIdentifier.replace(categoryName + ":", "")); + if (rowId != 0) + { + rowIds.add(rowId); + rowIdIdentifierMap.put(rowId, resourceIdentifier); + } + } + + Map> searchJsonMap = new HashMap<>(); + for (ExpMaterial material : ExperimentService.get().getExpMaterials(rowIds)) + { + searchJsonMap.put( + rowIdIdentifierMap.get(material.getRowId()), + ExperimentJSONConverter.serializeMaterial(material, user, ExperimentJSONConverter.DEFAULT_SETTINGS).toMap() + ); + } + + return searchJsonMap; + } + }); + } + + @Override + protected void startupAfterSpringConfig(ModuleContext moduleContext) + { + SearchService ss = SearchService.get(); +// ss.addSearchCategory(OntologyManager.conceptCategory); + ss.addSearchCategory(ExpSampleTypeImpl.searchCategory); + ss.addSearchCategory(ExpSampleTypeImpl.mediaSearchCategory); + ss.addSearchCategory(ExpMaterialImpl.searchCategory); + ss.addSearchCategory(ExpMaterialImpl.mediaSearchCategory); + ss.addSearchCategory(ExpDataClassImpl.SEARCH_CATEGORY); + ss.addSearchCategory(ExpDataClassImpl.MEDIA_SEARCH_CATEGORY); + ss.addSearchCategory(ExpDataImpl.expDataCategory); + ss.addSearchCategory(ExpDataImpl.expMediaDataCategory); + ss.addSearchResultTemplate(new ExpDataImpl.DataSearchResultTemplate()); + addDataResourceResolver(ExpDataImpl.expDataCategory.getName()); + addDataResourceResolver(ExpDataImpl.expMediaDataCategory.getName()); + addDataClassResourceResolver(ExpDataClassImpl.SEARCH_CATEGORY.getName()); + addDataClassResourceResolver(ExpDataClassImpl.MEDIA_SEARCH_CATEGORY.getName()); + addSampleTypeResourceResolver(ExpSampleTypeImpl.searchCategory.getName()); + addSampleTypeResourceResolver(ExpSampleTypeImpl.mediaSearchCategory.getName()); + addSampleResourceResolver(ExpMaterialImpl.searchCategory.getName()); + addSampleResourceResolver(ExpMaterialImpl.mediaSearchCategory.getName()); + ss.addDocumentProvider(ExperimentServiceImpl.get()); + + PipelineService.get().registerPipelineProvider(new ExperimentPipelineProvider(this)); + ExperimentService.get().registerExperimentRunTypeSource(container -> Collections.singleton(ExperimentRunType.ALL_RUNS_TYPE)); + ExperimentService.get().registerDataType(new LogDataType()); + + AuditLogService.get().registerAuditType(new DomainAuditProvider()); + AuditLogService.get().registerAuditType(new DomainPropertyAuditProvider()); + AuditLogService.get().registerAuditType(new ExperimentAuditProvider()); + AuditLogService.get().registerAuditType(new SampleTypeAuditProvider()); + AuditLogService.get().registerAuditType(new SampleTimelineAuditProvider()); + + FileContentService fileContentService = FileContentService.get(); + if (null != fileContentService) + { + fileContentService.addFileListener(new ExpDataFileListener()); + fileContentService.addFileListener(new TableUpdaterFileListener(ExperimentService.get().getTinfoExperimentRun(), "FilePathRoot", TableUpdaterFileListener.Type.fileRootPath, "RowId")); + fileContentService.addFileListener(new FileLinkFileListener()); + } + ContainerManager.addContainerListener(new ContainerManager.ContainerListener() + { + @Override + public void containerDeleted(Container c, User user) + { + try + { + ExperimentService.get().deleteAllExpObjInContainer(c, user); + } + catch (ExperimentException ee) + { + throw new RuntimeException(ee); + } + } + }, + // This is in the Last group because when a container is deleted, + // the Experiment listener needs to be called after the Study listener, + // because Study needs the metadata held by Experiment to delete properly. + // but it should be before the CoreContainerListener + ContainerManager.ContainerListener.Order.Last); + + if (ModuleLoader.getInstance().shouldInsertData()) + SystemProperty.registerProperties(); + + FolderSerializationRegistry folderRegistry = FolderSerializationRegistry.get(); + if (null != folderRegistry) + { + folderRegistry.addFactories(new FolderXarWriterFactory(), new FolderXarImporterFactory()); + folderRegistry.addWriterFactory(new SampleTypeFolderWriter.SampleTypeDesignWriter.Factory()); + folderRegistry.addWriterFactory(new SampleTypeFolderWriter.SampleTypeDataWriter.Factory()); + folderRegistry.addWriterFactory(new DataClassFolderWriter.DataClassDesignWriter.Factory()); + folderRegistry.addWriterFactory(new DataClassFolderWriter.DataClassDataWriter.Factory()); + folderRegistry.addImportFactory(new SampleTypeFolderImporter.Factory()); + folderRegistry.addImportFactory(new DataClassFolderImporter.Factory()); + folderRegistry.addImportFactory(new SampleStatusFolderImporter.Factory()); + } + + AttachmentService.get().registerAttachmentType(ExpDataClassType.get()); + + WebdavService.get().addProvider(new ScriptsResourceProvider()); + + SystemMaintenance.addTask(new FileLinkMetricsMaintenanceTask()); + + UsageMetricsService svc = UsageMetricsService.get(); + if (null != svc) + { + svc.registerUsageMetrics(getName(), () -> { + Map results = new HashMap<>(); + + DbSchema schema = ExperimentService.get().getSchema(); + if (AssayService.get() != null) + { + Map assayMetrics = new HashMap<>(); + SQLFragment baseRunSQL = new SQLFragment("SELECT COUNT(*) FROM ").append(ExperimentService.get().getTinfoExperimentRun(), "r").append(" WHERE lsid LIKE ?"); + SQLFragment baseProtocolSQL = new SQLFragment("SELECT * FROM ").append(ExperimentService.get().getTinfoProtocol(), "p").append(" WHERE lsid LIKE ? AND ApplicationType = ?"); + for (AssayProvider assayProvider : AssayService.get().getAssayProviders()) + { + Map protocolMetrics = new HashMap<>(); + + // Run count across all assay designs of this type + SQLFragment runSQL = new SQLFragment(baseRunSQL); + runSQL.add(Lsid.namespaceLikeString(assayProvider.getRunLSIDPrefix())); + protocolMetrics.put("runCount", new SqlSelector(schema, runSQL).getObject(Long.class)); + + // Number of assay designs of this type + SQLFragment protocolSQL = new SQLFragment(baseProtocolSQL); + protocolSQL.add(assayProvider.getProtocolPattern()); + protocolSQL.add(ExpProtocol.ApplicationType.ExperimentRun.toString()); + List protocols = new SqlSelector(schema, protocolSQL).getArrayList(Protocol.class); + protocolMetrics.put("protocolCount", protocols.size()); + + List wrappedProtocols = protocols.stream().map(ExpProtocolImpl::new).collect(Collectors.toList()); + + protocolMetrics.put("resultRowCount", assayProvider.getResultRowCount(wrappedProtocols)); + + // Primary implementation class + protocolMetrics.put("implementingClass", assayProvider.getClass()); + + assayMetrics.put(assayProvider.getName(), protocolMetrics); + } + assayMetrics.put("autoLinkedAssayCount", new SqlSelector(schema, "SELECT COUNT(*) FROM exp.protocol EP JOIN exp.objectPropertiesView OP ON EP.lsid = OP.objecturi WHERE OP.propertyuri = 'terms.labkey.org#AutoCopyTargetContainer'").getObject(Long.class)); + assayMetrics.put("protocolsWithTransformScriptCount", new SqlSelector(schema, "SELECT COUNT(*) FROM exp.protocol EP JOIN exp.objectPropertiesView OP ON EP.lsid = OP.objecturi WHERE OP.name = 'TransformScript' AND status = 'Active'").getObject(Long.class)); + assayMetrics.put("protocolsWithTransformScriptRunOnEditCount", new SqlSelector(schema, "SELECT COUNT(*) FROM exp.protocol EP JOIN exp.objectPropertiesView OP ON EP.lsid = OP.objecturi WHERE OP.name = 'TransformScript' AND status = 'Active' AND OP.stringvalue LIKE '%\"INSERT\"%'").getObject(Long.class)); + assayMetrics.put("protocolsWithTransformScriptRunOnImportCount", new SqlSelector(schema, "SELECT COUNT(*) FROM exp.protocol EP JOIN exp.objectPropertiesView OP ON EP.lsid = OP.objecturi WHERE OP.name = 'TransformScript' AND status = 'Active' AND OP.stringvalue LIKE '%\"INSERT\"%'").getObject(Long.class)); + + assayMetrics.put("standardAssayWithPlateSupportCount", new SqlSelector(schema, "SELECT COUNT(*) FROM exp.protocol EP JOIN exp.objectPropertiesView OP ON EP.lsid = OP.objecturi WHERE OP.name = 'PlateMetadata' AND floatValue = 1").getObject(Long.class)); + SQLFragment runsWithPlateSQL = new SQLFragment(""" + SELECT COUNT(*) FROM exp.experimentrun r + INNER JOIN exp.object o ON o.objectUri = r.lsid + INNER JOIN exp.objectproperty op ON op.objectId = o.objectId + WHERE op.propertyid IN ( + SELECT propertyid FROM exp.propertydescriptor WHERE name = ? AND lookupquery = ? + )"""); + assayMetrics.put("standardAssayRunsWithPlateTemplate", new SqlSelector(schema, new SQLFragment(runsWithPlateSQL).add("PlateTemplate").add("PlateTemplate")).getObject(Long.class)); + assayMetrics.put("standardAssayRunsWithPlateSet", new SqlSelector(schema, new SQLFragment(runsWithPlateSQL).add("PlateSet").add("PlateSet")).getObject(Long.class)); + + assayMetrics.put("assayRunsFileColumnCount", new SqlSelector(schema, """ + SELECT COUNT(DISTINCT DD.DomainURI) FROM + exp.PropertyDescriptor D\s + JOIN exp.PropertyDomain PD ON D.propertyId = PD.propertyid + JOIN exp.DomainDescriptor DD on PD.domainID = DD.domainId + WHERE DD.domainUri LIKE ? AND D.rangeURI = ?""", "urn:lsid:%:" + ExpProtocol.AssayDomainTypes.Run.getPrefix() + ".%", PropertyType.FILE_LINK.getTypeUri()).getObject(Long.class)); + + assayMetrics.put("assayResultsFileColumnCount", new SqlSelector(schema, """ + SELECT COUNT(DISTINCT DD.DomainURI) FROM + exp.PropertyDescriptor D\s + JOIN exp.PropertyDomain PD ON D.propertyId = PD.propertyid + JOIN exp.DomainDescriptor DD on PD.domainID = DD.domainId + WHERE DD.domainUri LIKE ? AND D.rangeURI = ?""", "urn:lsid:%:" + ExpProtocol.AssayDomainTypes.Result.getPrefix() + ".%", PropertyType.FILE_LINK.getTypeUri()).getObject(Long.class)); + + Map sampleLookupCountMetrics = new HashMap<>(); + SQLFragment baseAssaySampleLookupSQL = new SQLFragment("SELECT COUNT(*) FROM exp.propertydescriptor WHERE (lookupschema = 'samples' OR (lookupschema = 'exp' AND lookupquery = 'Materials')) AND propertyuri LIKE ?"); + + SQLFragment batchAssaySampleLookupSQL = new SQLFragment(baseAssaySampleLookupSQL); + batchAssaySampleLookupSQL.add("urn:lsid:%:" + ExpProtocol.AssayDomainTypes.Batch.getPrefix() + ".%"); + sampleLookupCountMetrics.put("batchDomain", new SqlSelector(schema, batchAssaySampleLookupSQL).getObject(Long.class)); + + SQLFragment runAssaySampleLookupSQL = new SQLFragment(baseAssaySampleLookupSQL); + runAssaySampleLookupSQL.add("urn:lsid:%:" + ExpProtocol.AssayDomainTypes.Run.getPrefix() + ".%"); + sampleLookupCountMetrics.put("runDomain", new SqlSelector(schema, runAssaySampleLookupSQL).getObject(Long.class)); + + SQLFragment resultAssaySampleLookupSQL = new SQLFragment(baseAssaySampleLookupSQL); + resultAssaySampleLookupSQL.add("urn:lsid:%:" + ExpProtocol.AssayDomainTypes.Result.getPrefix() + ".%"); + sampleLookupCountMetrics.put("resultDomain", new SqlSelector(schema, resultAssaySampleLookupSQL).getObject(Long.class)); + + SQLFragment resultAssayMultipleSampleLookupSQL = new SQLFragment( + """ + SELECT COUNT(*) FROM ( + SELECT PD.domainid, COUNT(*) AS PropCount + FROM exp.propertydescriptor D + JOIN exp.PropertyDomain PD ON D.propertyId = PD.propertyid + WHERE (lookupschema = 'samples' OR (lookupschema = 'exp' AND lookupquery = 'Materials')) + AND propertyuri LIKE ? + GROUP BY PD.domainid + ) X WHERE X.PropCount > 1""" + ); + resultAssayMultipleSampleLookupSQL.add("urn:lsid:%:" + ExpProtocol.AssayDomainTypes.Result.getPrefix() + ".%"); + sampleLookupCountMetrics.put("resultDomainWithMultiple", new SqlSelector(schema, resultAssayMultipleSampleLookupSQL).getObject(Long.class)); + + assayMetrics.put("sampleLookupCount", sampleLookupCountMetrics); + + + // Putting these metrics at the same level as the other BooleanColumnCount metrics (e.g., sampleTypeWithBooleanColumnCount) + results.put("assayResultWithBooleanColumnCount", new SqlSelector(schema, """ + SELECT COUNT(DISTINCT DD.DomainURI) FROM + exp.PropertyDescriptor D\s + JOIN exp.PropertyDomain PD ON D.propertyId = PD.propertyid + JOIN exp.DomainDescriptor DD on PD.domainID = DD.domainId + WHERE D.propertyURI LIKE ? AND D.rangeURI = ?""", "urn:lsid:%:" + ExpProtocol.AssayDomainTypes.Result.getPrefix() + ".%", PropertyType.BOOLEAN.getTypeUri()).getObject(Long.class)); + + results.put("assayRunWithBooleanColumnCount", new SqlSelector(schema, """ + SELECT COUNT(DISTINCT DD.DomainURI) FROM + exp.PropertyDescriptor D\s + JOIN exp.PropertyDomain PD ON D.propertyId = PD.propertyid + JOIN exp.DomainDescriptor DD on PD.domainID = DD.domainId + WHERE D.propertyURI LIKE ? AND D.rangeURI = ?""", "urn:lsid:%:" + ExpProtocol.AssayDomainTypes.Run.getPrefix() + ".%", PropertyType.BOOLEAN.getTypeUri()).getObject(Long.class)); + + results.put("assay", assayMetrics); + } + + results.put("autoLinkedSampleSetCount", new SqlSelector(schema, "SELECT COUNT(*) FROM exp.materialsource WHERE autoLinkTargetContainer IS NOT NULL").getObject(Long.class)); + results.put("sampleSetCount", new SqlSelector(schema, "SELECT COUNT(*) FROM exp.materialsource").getObject(Long.class)); + + if (schema.getSqlDialect().isPostgreSQL()) // SQLServer does not support regular expression queries + { + Collection> numSampleCounts = new SqlSelector(schema, """ + SELECT totalCount, numberNameCount FROM + (SELECT cpastype, COUNT(*) AS totalCount from exp.material GROUP BY cpastype) t + JOIN + (SELECT cpastype, COUNT(*) AS numberNameCount FROM exp.material m WHERE m.name SIMILAR TO '[0-9.]*' GROUP BY cpastype) ns + ON t.cpastype = ns.cpastype""").getMapCollection(); + results.put("sampleSetWithNumberNamesCount", numSampleCounts.size()); + results.put("sampleSetWithOnlyNumberNamesCount", numSampleCounts.stream().filter( + map -> (Long) map.get("totalCount") > 0 && map.get("totalCount") == map.get("numberNameCount") + ).count()); + } + UserSchema userSchema = AuditLogService.getAuditLogSchema(User.getSearchUser(), ContainerManager.getRoot()); + FilteredTable table = (FilteredTable) userSchema.getTable(SampleTimelineAuditEvent.EVENT_TYPE); + + SQLFragment sql = new SQLFragment("SELECT COUNT(*)\n" + + " FROM (\n" + + " -- updates that are marked as lineage updates\n" + + " (SELECT DISTINCT transactionId\n" + + " FROM " + table.getRealTable().getFromSQL("").getSQL() +"\n" + + " WHERE islineageupdate = " + schema.getSqlDialect().getBooleanTRUE() + "\n" + + " AND comment = 'Sample was updated.'\n" + + " ) a1\n" + + " JOIN\n" + + " -- but have associated entries that are not lineage updates\n" + + " (SELECT DISTINCT transactionid\n" + + " FROM " + table.getRealTable().getFromSQL("").getSQL() + "\n" + + " WHERE islineageupdate = " + schema.getSqlDialect().getBooleanFALSE() + ") a2\n" + + " ON a1.transactionid = a2.transactionid\n" + + " )"); + + results.put("sampleLineageAuditDiscrepancyCount", new SqlSelector(schema, sql.getSQL()).getObject(Long.class)); + + results.put("sampleCount", new SqlSelector(schema, "SELECT COUNT(*) FROM exp.material").getObject(Long.class)); + results.put("aliquotCount", new SqlSelector(schema, "SELECT COUNT(*) FROM exp.material where aliquotedfromlsid IS NOT NULL").getObject(Long.class)); + results.put("sampleNullAmountCount", new SqlSelector(schema, "SELECT COUNT(*) FROM exp.material WHERE storedamount IS NULL").getObject(Long.class)); + results.put("sampleNegativeAmountCount", new SqlSelector(schema, "SELECT COUNT(*) FROM exp.material WHERE storedamount < 0").getObject(Long.class)); + results.put("sampleUnitsDifferCount", new SqlSelector(schema, "SELECT COUNT(*) from exp.material m JOIN exp.materialSource s ON m.materialsourceid = s.rowid WHERE m.units != s.metricunit").getObject(Long.class)); + results.put("sampleTypesWithoutUnitsCount", new SqlSelector(schema, "SELECT COUNT(*) from exp.materialSource WHERE category IS NULL AND metricunit IS NULL").getObject(Long.class)); + + results.put("duplicateSampleMaterialNameCount", new SqlSelector(schema, "SELECT COUNT(*) as duplicateCount FROM " + + "(SELECT name, cpastype FROM exp.material WHERE cpastype <> 'Material' GROUP BY name, cpastype HAVING COUNT(*) > 1) d").getObject(Long.class)); + results.put("duplicateSpecimenMaterialNameCount", new SqlSelector(schema, "SELECT COUNT(*) as duplicateCount FROM " + + "(SELECT name, cpastype FROM exp.material WHERE cpastype = 'Material' GROUP BY name, cpastype HAVING COUNT(*) > 1) d").getObject(Long.class)); + String duplicateCaseInsensitiveSampleNameCountSql = """ + SELECT COUNT(*) FROM + ( + SELECT 1 AS found + FROM exp.material + WHERE materialsourceid IS NOT NULL + GROUP BY LOWER(name), materialsourceid + HAVING COUNT(*) > 1 + ) AS duplicates + """; + String duplicateCaseInsensitiveDataNameCountSql = """ + SELECT COUNT(*) FROM + ( + SELECT 1 AS found + FROM exp.data + WHERE classid IS NOT NULL + GROUP BY LOWER(name), classid + HAVING COUNT(*) > 1 + ) AS duplicates + """; + results.put("duplicateCaseInsensitiveSampleNameCount", new SqlSelector(schema, duplicateCaseInsensitiveSampleNameCountSql).getObject(Long.class)); + results.put("duplicateCaseInsensitiveDataNameCount", new SqlSelector(schema, duplicateCaseInsensitiveDataNameCountSql).getObject(Long.class)); + + results.put("dataClassCount", new SqlSelector(schema, "SELECT COUNT(*) FROM exp.dataclass").getObject(Long.class)); + results.put("dataClassRowCount", new SqlSelector(schema, "SELECT COUNT(*) FROM exp.data WHERE classid IN (SELECT rowid FROM exp.dataclass)").getObject(Long.class)); + results.put("dataWithDataParentsCount", new SqlSelector(schema, "SELECT COUNT(DISTINCT d.sourceApplicationId) FROM exp.data d\n" + + "JOIN exp.datainput di ON di.targetapplicationid = d.sourceapplicationid").getObject(Long.class)); + if (schema.getSqlDialect().isPostgreSQL()) + { + Collection> numDataClassObjectsCounts = new SqlSelector(schema, """ + SELECT totalCount, numberNameCount FROM + (SELECT cpastype, COUNT(*) AS totalCount from exp.data GROUP BY cpastype) t + JOIN + (SELECT cpastype, COUNT(*) AS numberNameCount FROM exp.data m WHERE m.name SIMILAR TO '[0-9.]*' GROUP BY cpastype) ns + ON t.cpastype = ns.cpastype""").getMapCollection(); + results.put("dataClassWithNumberNamesCount", numDataClassObjectsCounts.size()); + results.put("dataClassWithOnlyNumberNamesCount", numDataClassObjectsCounts.stream().filter(map -> + (Long) map.get("totalCount") > 0 && map.get("totalCount") == map.get("numberNameCount")).count()); + } + + results.put("ontologyPrincipalConceptCodeCount", new SqlSelector(schema, "SELECT COUNT(*) FROM exp.propertydescriptor WHERE principalconceptcode IS NOT NULL").getObject(Long.class)); + results.put("ontologyLookupColumnCount", new SqlSelector(schema, "SELECT COUNT(*) FROM exp.propertydescriptor WHERE concepturi = ?", OntologyService.conceptCodeConceptURI).getObject(Long.class)); + results.put("ontologyConceptSubtreeCount", new SqlSelector(schema, "SELECT COUNT(*) FROM exp.propertydescriptor WHERE conceptsubtree IS NOT NULL").getObject(Long.class)); + results.put("ontologyConceptImportColumnCount", new SqlSelector(schema, "SELECT COUNT(*) FROM exp.propertydescriptor WHERE conceptimportcolumn IS NOT NULL").getObject(Long.class)); + results.put("ontologyConceptLabelColumnCount", new SqlSelector(schema, "SELECT COUNT(*) FROM exp.propertydescriptor WHERE conceptlabelcolumn IS NOT NULL").getObject(Long.class)); + + results.put("scannableColumnCount", new SqlSelector(schema, "SELECT COUNT(*) FROM exp.propertydescriptor WHERE scannable = ?", true).getObject(Long.class)); + results.put("uniqueIdColumnCount", new SqlSelector(schema, "SELECT COUNT(*) FROM exp.propertydescriptor WHERE concepturi = ?", STORAGE_UNIQUE_ID_CONCEPT_URI).getObject(Long.class)); + results.put("sampleTypeWithUniqueIdCount", new SqlSelector(schema, """ + SELECT COUNT(DISTINCT DD.DomainURI) FROM + exp.PropertyDescriptor D\s + JOIN exp.PropertyDomain PD ON D.propertyId = PD.propertyid + JOIN exp.DomainDescriptor DD on PD.domainID = DD.domainId + WHERE D.conceptURI = ?""", STORAGE_UNIQUE_ID_CONCEPT_URI).getObject(Long.class)); + + results.put("fileColumnCount", new SqlSelector(schema, "SELECT COUNT(*) FROM exp.propertydescriptor WHERE rangeURI = ?", PropertyType.FILE_LINK.getTypeUri()).getObject(Long.class)); + results.put("sampleTypeWithFileColumnCount", new SqlSelector(schema, """ + SELECT COUNT(DISTINCT DD.DomainURI) FROM + exp.PropertyDescriptor D\s + JOIN exp.PropertyDomain PD ON D.propertyId = PD.propertyid + JOIN exp.DomainDescriptor DD on PD.domainID = DD.domainId + WHERE DD.storageSchemaName = ? AND D.rangeURI = ?""", SampleTypeDomainKind.PROVISIONED_SCHEMA_NAME, PropertyType.FILE_LINK.getTypeUri()).getObject(Long.class)); + results.put("sampleTypeWithBooleanColumnCount", new SqlSelector(schema, """ + SELECT COUNT(DISTINCT DD.DomainURI) FROM + exp.PropertyDescriptor D\s + JOIN exp.PropertyDomain PD ON D.propertyId = PD.propertyid + JOIN exp.DomainDescriptor DD on PD.domainID = DD.domainId + WHERE DD.storageSchemaName = ? AND D.rangeURI = ?""", SampleTypeDomainKind.PROVISIONED_SCHEMA_NAME, PropertyType.BOOLEAN.getTypeUri()).getObject(Long.class)); + + results.put("sampleTypeAliquotSpecificField", new SqlSelector(schema, """ + SELECT COUNT(DISTINCT D.PropertyURI) FROM + exp.PropertyDescriptor D\s + JOIN exp.PropertyDomain PD ON D.propertyId = PD.propertyid + JOIN exp.DomainDescriptor DD on PD.domainID = DD.domainId + WHERE DD.storageSchemaName = ? AND D.derivationDataScope = ?""", SampleTypeDomainKind.PROVISIONED_SCHEMA_NAME, ExpSchema.DerivationDataScopeType.ChildOnly.name()).getObject(Long.class)); + results.put("sampleTypeParentOnlyField", new SqlSelector(schema, """ + SELECT COUNT(DISTINCT D.PropertyURI) FROM + exp.PropertyDescriptor D\s + JOIN exp.PropertyDomain PD ON D.propertyId = PD.propertyid + JOIN exp.DomainDescriptor DD on PD.domainID = DD.domainId + WHERE DD.storageSchemaName = ? AND (D.derivationDataScope = ? OR D.derivationDataScope IS NULL)""", SampleTypeDomainKind.PROVISIONED_SCHEMA_NAME, ExpSchema.DerivationDataScopeType.ParentOnly.name()).getObject(Long.class)); + results.put("sampleTypeParentAndAliquotField", new SqlSelector(schema, """ + SELECT COUNT(DISTINCT D.PropertyURI) FROM + exp.PropertyDescriptor D\s + JOIN exp.PropertyDomain PD ON D.propertyId = PD.propertyid + JOIN exp.DomainDescriptor DD on PD.domainID = DD.domainId + WHERE DD.storageSchemaName = ? AND D.derivationDataScope = ?""", SampleTypeDomainKind.PROVISIONED_SCHEMA_NAME, ExpSchema.DerivationDataScopeType.All.name()).getObject(Long.class)); + + results.put("attachmentColumnCount", new SqlSelector(schema, "SELECT COUNT(*) FROM exp.propertydescriptor WHERE rangeURI = ?", PropertyType.ATTACHMENT.getTypeUri()).getObject(Long.class)); + results.put("dataClassWithAttachmentColumnCount", new SqlSelector(schema, """ + SELECT COUNT(DISTINCT DD.DomainURI) FROM + exp.PropertyDescriptor D\s + JOIN exp.PropertyDomain PD ON D.propertyId = PD.propertyid + JOIN exp.DomainDescriptor DD on PD.domainID = DD.domainId + WHERE DD.storageSchemaName = ? AND D.rangeURI = ?""", DataClassDomainKind.PROVISIONED_SCHEMA_NAME, PropertyType.ATTACHMENT.getTypeUri()).getObject(Long.class)); + results.put("dataClassWithBooleanColumnCount", new SqlSelector(schema, """ + SELECT COUNT(DISTINCT DD.DomainURI) FROM + exp.PropertyDescriptor D\s + JOIN exp.PropertyDomain PD ON D.propertyId = PD.propertyid + JOIN exp.DomainDescriptor DD on PD.domainID = DD.domainId + WHERE DD.storageSchemaName = ? AND D.rangeURI = ?""", DataClassDomainKind.PROVISIONED_SCHEMA_NAME, PropertyType.BOOLEAN.getTypeUri()).getObject(Long.class)); + + results.put("textChoiceColumnCount", new SqlSelector(schema, "SELECT COUNT(*) FROM exp.propertydescriptor WHERE concepturi = ?", TEXT_CHOICE_CONCEPT_URI).getObject(Long.class)); + + results.put("domainsWithDateTimeColumnCount", new SqlSelector(schema, """ + SELECT COUNT(DISTINCT DD.DomainURI) FROM + exp.PropertyDescriptor D\s + JOIN exp.PropertyDomain PD ON D.propertyId = PD.propertyid + JOIN exp.DomainDescriptor DD on PD.domainID = DD.domainId + WHERE D.rangeURI = ?""", PropertyType.DATE_TIME.getTypeUri()).getObject(Long.class)); + + results.put("domainsWithDateColumnCount", new SqlSelector(schema, """ + SELECT COUNT(DISTINCT DD.DomainURI) FROM + exp.PropertyDescriptor D\s + JOIN exp.PropertyDomain PD ON D.propertyId = PD.propertyid + JOIN exp.DomainDescriptor DD on PD.domainID = DD.domainId + WHERE D.rangeURI = ?""", PropertyType.DATE.getTypeUri()).getObject(Long.class)); + + results.put("domainsWithTimeColumnCount", new SqlSelector(schema, """ + SELECT COUNT(DISTINCT DD.DomainURI) FROM + exp.PropertyDescriptor D\s + JOIN exp.PropertyDomain PD ON D.propertyId = PD.propertyid + JOIN exp.DomainDescriptor DD on PD.domainID = DD.domainId + WHERE D.rangeURI = ?""", PropertyType.TIME.getTypeUri()).getObject(Long.class)); + + results.put("maxObjectObjectId", new SqlSelector(schema, "SELECT MAX(ObjectId) FROM exp.Object").getObject(Long.class)); + results.put("maxMaterialRowId", new SqlSelector(schema, "SELECT MAX(RowId) FROM exp.Material").getObject(Long.class)); + + results.putAll(ExperimentService.get().getDomainMetrics()); + + return results; + }); + } + + ExperimentMigrationSchemaHandler handler = new ExperimentMigrationSchemaHandler(); + DatabaseMigrationService.get().registerSchemaHandler(handler); + DatabaseMigrationService.get().registerTableHandler(new MigrationTableHandler() + { + @Override + public TableInfo getTableInfo() + { + return DbSchema.get("premium", DbSchemaType.Bare).getTable("Exclusions"); + } + + @Override + public void adjustFilter(TableInfo sourceTable, SimpleFilter filter, Set containers) + { + // Include experiment runs that were copied + FilterClause includedClause = handler.getIncludedRowIdClause(sourceTable, FieldKey.fromParts("RunId")); + if (includedClause != null) + filter.addClause(includedClause); + } + }); + DatabaseMigrationService.get().registerTableHandler(new MigrationTableHandler() + { + @Override + public TableInfo getTableInfo() + { + return DbSchema.get("premium", DbSchemaType.Bare).getTable("ExclusionMaps"); + } + + @Override + public void adjustFilter(TableInfo sourceTable, SimpleFilter filter, Set containers) + { + // Include experiment runs that were copied + FilterClause includedClause = handler.getIncludedRowIdClause(sourceTable, FieldKey.fromParts("ExclusionId", "RunId")); + if (includedClause != null) + filter.addClause(includedClause); + } + }); + DatabaseMigrationService.get().registerTableHandler(new MigrationTableHandler() + { + @Override + public TableInfo getTableInfo() + { + return DbSchema.get("assayrequest", DbSchemaType.Bare).getTable("RequestRunsJunction"); + } + + @Override + public void adjustFilter(TableInfo sourceTable, SimpleFilter filter, Set containers) + { + // Include experiment runs that were copied + FilterClause includedClause = handler.getIncludedRowIdClause(sourceTable, FieldKey.fromParts("RunId")); + if (includedClause != null) + filter.addClause(includedClause); + } + }); + DatabaseMigrationService.get().registerSchemaHandler(new SampleTypeMigrationSchemaHandler()); + DataClassMigrationSchemaHandler dcHandler = new DataClassMigrationSchemaHandler(); + DatabaseMigrationService.get().registerSchemaHandler(dcHandler); + ExperimentDeleteService.setInstance(dcHandler); + } + + @Override + @NotNull + public Collection getSummary(Container c) + { + Collection list = new LinkedList<>(); + int runGroupCount = ExperimentService.get().getExperiments(c, null, false, true).size(); + if (runGroupCount > 0) + list.add(StringUtilsLabKey.pluralize(runGroupCount, "Run Group")); + + User user = HttpView.currentContext().getUser(); + + Set runTypes = ExperimentService.get().getExperimentRunTypes(c); + for (ExperimentRunType runType : runTypes) + { + if (runType == ExperimentRunType.ALL_RUNS_TYPE) + continue; + + long runCount = runType.getRunCount(user, c); + if (runCount > 0) + list.add(runCount + " runs of type " + runType.getDescription()); + } + + int dataClassCount = ExperimentService.get().getDataClasses(c, null, false).size(); + if (dataClassCount > 0) + list.add(dataClassCount + " Data Class" + (dataClassCount > 1 ? "es" : "")); + + int sampleTypeCount = SampleTypeService.get().getSampleTypes(c, false).size(); + if (sampleTypeCount > 0) + list.add(sampleTypeCount + " Sample Type" + (sampleTypeCount > 1 ? "s" : "")); + + return list; + } + + @Override + public @NotNull ArrayList getDetailedSummary(Container c, User user) + { + ArrayList summaries = new ArrayList<>(); + + // Assay types + long assayTypeCount = AssayService.get().getAssayProtocols(c).stream().filter(p -> p.getContainer().equals(c)).count(); + if (assayTypeCount > 0) + summaries.add(new Summary(assayTypeCount, "Assay Type")); + + // Run count + int runGroupCount = ExperimentService.get().getExperiments(c, user, false, true).size(); + if (runGroupCount > 0) + summaries.add(new Summary(runGroupCount, "Assay run")); + + // Number of Data Classes + List dataClasses = ExperimentService.get().getDataClasses(c, user, false); + int dataClassCount = dataClasses.size(); + if (dataClassCount > 0) + summaries.add(new Summary(dataClassCount, "Data Class")); + + ExpSchema expSchema = new ExpSchema(user, c); + + // Individual Data Class row counts + { + // The table-level container filter is set to ensure data class types are included + // that may not be defined in the target container but may have rows of data in the target container + TableInfo table = ExpSchema.TableType.DataClasses.createTable(expSchema, null, ContainerFilter.Type.CurrentPlusProjectAndShared.create(c, user)); + + // Issue 47919: The "DataCount" column is filtered to only count data in the target container + if (table instanceof ExpDataClassTableImpl tableImpl) + tableImpl.setDataCountContainerFilter(ContainerFilter.Type.Current.create(c, user)); + + Set columns = new LinkedHashSet<>(); + columns.add(ExpDataClassTable.Column.Name.name()); + columns.add(ExpDataClassTable.Column.DataCount.name()); + + Map results = new TableSelector(table, columns).getValueMap(String.class); + for (var entry : results.entrySet()) + { + long count = entry.getValue().longValue(); + if (count > 0) + summaries.add(new Summary(count, entry.getKey())); + } + } + + // Sample Types + int sampleTypeCount = SampleTypeService.get().getSampleTypes(c, false).size(); + if (sampleTypeCount > 0) + summaries.add(new Summary(sampleTypeCount, "Sample Type")); + + // Individual Sample Type row counts + { + // The table-level container filter is set to ensure data class types are included + // that may not be defined in the target container but may have rows of data in the target container + TableInfo table = ExpSchema.TableType.SampleSets.createTable(expSchema, null, ContainerFilter.Type.CurrentPlusProjectAndShared.create(c, user)); + + // Issue 51557: The "SampleCount" column is filtered to only count data in the target container + if (table instanceof ExpSampleTypeTableImpl tableImpl) + tableImpl.setSampleCountContainerFilter(ContainerFilter.Type.Current.create(c, user)); + + Set columns = new LinkedHashSet<>(); + columns.add(ExpSampleTypeTable.Column.Name.name()); + columns.add(ExpSampleTypeTable.Column.SampleCount.name()); + + Map results = new TableSelector(table, columns).getValueMap(String.class); + for (var entry : results.entrySet()) + { + long count = entry.getValue().longValue(); + if (count > 0) + { + String name = entry.getKey(); + Summary s = name.equals("MixtureBatches") + ? new Summary(count, "Batch") + : new Summary(count, name); + summaries.add(s); + } + } + } + + return summaries; + } + + @Override + public @NotNull Set> getIntegrationTests() + { + return Set.of( + DomainImpl.TestCase.class, + DomainPropertyImpl.TestCase.class, + ExpDataTableImpl.TestCase.class, + ExperimentServiceImpl.AuditDomainUriTest.class, + ExperimentServiceImpl.LineageQueryTestCase.class, + ExperimentServiceImpl.ParseInputOutputAliasTestCase.class, + ExperimentServiceImpl.TestCase.class, + ExperimentStressTest.class, + LineagePerfTest.class, + LineageTest.class, + OntologyManager.TestCase.class, + PropertyServiceImpl.TestCase.class, + SampleTypeServiceImpl.TestCase.class, + StorageNameGenerator.TestCase.class, + StorageProvisionerImpl.TestCase.class, + UniqueValueCounterTestCase.class, + XarTestPipelineJob.TestCase.class + ); + } + + @Override + public @NotNull Collection>> getIntegrationTestFactories() + { + List>> list = new ArrayList<>(super.getIntegrationTestFactories()); + list.add(new JspTestCase("/org/labkey/experiment/api/ExpDataClassDataTestCase.jsp")); + list.add(new JspTestCase("/org/labkey/experiment/api/ExpSampleTypeTestCase.jsp")); + return list; + } + + @Override + public @NotNull Set> getUnitTests() + { + return Set.of( + GraphAlgorithms.TestCase.class, + LSIDRelativizer.TestCase.class, + Lsid.TestCase.class, + LsidUtils.TestCase.class, + PropertyController.TestCase.class, + Quantity.TestCase.class, + Unit.TestCase.class + ); + } + + @Override + @NotNull + public Collection getSchemaNames() + { + return List.of( + ExpSchema.SCHEMA_NAME, + DataClassDomainKind.PROVISIONED_SCHEMA_NAME, + SampleTypeDomainKind.PROVISIONED_SCHEMA_NAME + ); + } + + @NotNull + @Override + public Collection getProvisionedSchemaNames() + { + return PageFlowUtil.set(DataClassDomainKind.PROVISIONED_SCHEMA_NAME, SampleTypeDomainKind.PROVISIONED_SCHEMA_NAME); + } +} From 9031881ef999d571b5bd5812507e7d20367e9f82 Mon Sep 17 00:00:00 2001 From: XingY Date: Fri, 9 Jan 2026 09:27:00 -0800 Subject: [PATCH 05/14] crlf --- .../labkey/experiment/ExperimentModule.java | 2248 ++++++++--------- 1 file changed, 1124 insertions(+), 1124 deletions(-) diff --git a/experiment/src/org/labkey/experiment/ExperimentModule.java b/experiment/src/org/labkey/experiment/ExperimentModule.java index f29a5ab73b2..466c4f437c2 100644 --- a/experiment/src/org/labkey/experiment/ExperimentModule.java +++ b/experiment/src/org/labkey/experiment/ExperimentModule.java @@ -1,1124 +1,1124 @@ -/* - * Copyright (c) 2008-2019 LabKey Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.labkey.experiment; - -import org.apache.commons.lang3.math.NumberUtils; -import org.jetbrains.annotations.NotNull; -import org.jetbrains.annotations.Nullable; -import org.labkey.api.admin.FolderSerializationRegistry; -import org.labkey.api.assay.AssayProvider; -import org.labkey.api.assay.AssayService; -import org.labkey.api.attachments.AttachmentService; -import org.labkey.api.audit.AuditLogService; -import org.labkey.api.audit.SampleTimelineAuditEvent; -import org.labkey.api.collections.LongHashMap; -import org.labkey.api.data.Container; -import org.labkey.api.data.ContainerFilter; -import org.labkey.api.data.ContainerManager; -import org.labkey.api.data.CoreSchema; -import org.labkey.api.data.DbSchema; -import org.labkey.api.data.DbSchemaType; -import org.labkey.api.data.JdbcType; -import org.labkey.api.data.NameGenerator; -import org.labkey.api.data.SQLFragment; -import org.labkey.api.data.SimpleFilter; -import org.labkey.api.data.SimpleFilter.FilterClause; -import org.labkey.api.data.SqlSelector; -import org.labkey.api.data.TableInfo; -import org.labkey.api.data.TableSelector; -import org.labkey.api.data.UpgradeCode; -import org.labkey.api.defaults.DefaultValueService; -import org.labkey.api.exp.ExperimentException; -import org.labkey.api.exp.ExperimentRunType; -import org.labkey.api.exp.Lsid; -import org.labkey.api.exp.OntologyManager; -import org.labkey.api.exp.PropertyType; -import org.labkey.api.exp.api.DefaultExperimentDataHandler; -import org.labkey.api.exp.api.ExpData; -import org.labkey.api.exp.api.ExpDataClass; -import org.labkey.api.exp.api.ExpLineageService; -import org.labkey.api.exp.api.ExpMaterial; -import org.labkey.api.exp.api.ExpProtocol; -import org.labkey.api.exp.api.ExpProtocolAttachmentType; -import org.labkey.api.exp.api.ExpRunAttachmentType; -import org.labkey.api.exp.api.ExpSampleType; -import org.labkey.api.exp.api.ExperimentJSONConverter; -import org.labkey.api.exp.api.ExperimentService; -import org.labkey.api.exp.api.FilterProtocolInputCriteria; -import org.labkey.api.exp.api.SampleTypeDomainKind; -import org.labkey.api.exp.api.SampleTypeService; -import org.labkey.api.exp.api.StorageProvisioner; -import org.labkey.api.exp.property.DomainAuditProvider; -import org.labkey.api.exp.property.DomainPropertyAuditProvider; -import org.labkey.api.exp.property.ExperimentProperty; -import org.labkey.api.exp.property.PropertyService; -import org.labkey.api.exp.property.SystemProperty; -import org.labkey.api.exp.query.ExpDataClassTable; -import org.labkey.api.exp.query.ExpSampleTypeTable; -import org.labkey.api.exp.query.ExpSchema; -import org.labkey.api.exp.query.SamplesSchema; -import org.labkey.api.exp.xar.LSIDRelativizer; -import org.labkey.api.exp.xar.LsidUtils; -import org.labkey.api.files.FileContentService; -import org.labkey.api.files.TableUpdaterFileListener; -import org.labkey.api.migration.DatabaseMigrationService; -import org.labkey.api.migration.ExperimentDeleteService; -import org.labkey.api.migration.MigrationTableHandler; -import org.labkey.api.module.ModuleContext; -import org.labkey.api.module.ModuleLoader; -import org.labkey.api.module.SpringModule; -import org.labkey.api.module.Summary; -import org.labkey.api.ontology.OntologyService; -import org.labkey.api.ontology.Quantity; -import org.labkey.api.ontology.Unit; -import org.labkey.api.pipeline.PipelineService; -import org.labkey.api.query.FieldKey; -import org.labkey.api.query.FilteredTable; -import org.labkey.api.query.QueryService; -import org.labkey.api.query.UserSchema; -import org.labkey.api.search.SearchService; -import org.labkey.api.security.User; -import org.labkey.api.security.roles.RoleManager; -import org.labkey.api.settings.AppProps; -import org.labkey.api.settings.OptionalFeatureService; -import org.labkey.api.usageMetrics.UsageMetricsService; -import org.labkey.api.util.GUID; -import org.labkey.api.util.JspTestCase; -import org.labkey.api.util.PageFlowUtil; -import org.labkey.api.util.StringUtilsLabKey; -import org.labkey.api.util.SystemMaintenance; -import org.labkey.api.view.AlwaysAvailableWebPartFactory; -import org.labkey.api.view.BaseWebPartFactory; -import org.labkey.api.view.HttpView; -import org.labkey.api.view.JspView; -import org.labkey.api.view.Portal; -import org.labkey.api.view.ViewContext; -import org.labkey.api.view.WebPartFactory; -import org.labkey.api.view.WebPartView; -import org.labkey.api.view.template.WarningService; -import org.labkey.api.vocabulary.security.DesignVocabularyPermission; -import org.labkey.api.webdav.WebdavResource; -import org.labkey.api.webdav.WebdavService; -import org.labkey.experiment.api.DataClassDomainKind; -import org.labkey.experiment.api.ExpDataClassImpl; -import org.labkey.experiment.api.ExpDataClassTableImpl; -import org.labkey.experiment.api.ExpDataClassType; -import org.labkey.experiment.api.ExpDataImpl; -import org.labkey.experiment.api.ExpDataTableImpl; -import org.labkey.experiment.api.ExpMaterialImpl; -import org.labkey.experiment.api.ExpProtocolImpl; -import org.labkey.experiment.api.ExpSampleTypeImpl; -import org.labkey.experiment.api.ExpSampleTypeTableImpl; -import org.labkey.experiment.api.ExperimentServiceImpl; -import org.labkey.experiment.api.ExperimentStressTest; -import org.labkey.experiment.api.GraphAlgorithms; -import org.labkey.experiment.api.LineageTest; -import org.labkey.experiment.api.LogDataType; -import org.labkey.experiment.api.Protocol; -import org.labkey.experiment.api.SampleTypeServiceImpl; -import org.labkey.experiment.api.UniqueValueCounterTestCase; -import org.labkey.experiment.api.VocabularyDomainKind; -import org.labkey.experiment.api.data.ChildOfCompareType; -import org.labkey.experiment.api.data.ChildOfMethod; -import org.labkey.experiment.api.data.LineageCompareType; -import org.labkey.experiment.api.data.ParentOfCompareType; -import org.labkey.experiment.api.data.ParentOfMethod; -import org.labkey.experiment.api.property.DomainImpl; -import org.labkey.experiment.api.property.DomainPropertyImpl; -import org.labkey.experiment.api.property.LengthValidator; -import org.labkey.experiment.api.property.LookupValidator; -import org.labkey.experiment.api.property.PropertyServiceImpl; -import org.labkey.experiment.api.property.RangeValidator; -import org.labkey.experiment.api.property.RegExValidator; -import org.labkey.experiment.api.property.StorageNameGenerator; -import org.labkey.experiment.api.property.StorageProvisionerImpl; -import org.labkey.experiment.api.property.TextChoiceValidator; -import org.labkey.experiment.controllers.exp.ExperimentController; -import org.labkey.experiment.controllers.property.PropertyController; -import org.labkey.experiment.defaults.DefaultValueServiceImpl; -import org.labkey.experiment.lineage.ExpLineageServiceImpl; -import org.labkey.experiment.lineage.LineagePerfTest; -import org.labkey.experiment.pipeline.ExperimentPipelineProvider; -import org.labkey.experiment.pipeline.XarTestPipelineJob; -import org.labkey.experiment.samples.DataClassFolderImporter; -import org.labkey.experiment.samples.DataClassFolderWriter; -import org.labkey.experiment.samples.SampleStatusFolderImporter; -import org.labkey.experiment.samples.SampleTimelineAuditProvider; -import org.labkey.experiment.samples.SampleTypeFolderImporter; -import org.labkey.experiment.samples.SampleTypeFolderWriter; -import org.labkey.experiment.security.DataClassDesignerRole; -import org.labkey.experiment.security.SampleTypeDesignerRole; -import org.labkey.experiment.types.TypesController; -import org.labkey.experiment.xar.FolderXarImporterFactory; -import org.labkey.experiment.xar.FolderXarWriterFactory; - -import java.util.ArrayList; -import java.util.Collection; -import java.util.Collections; -import java.util.HashMap; -import java.util.HashSet; -import java.util.LinkedHashSet; -import java.util.LinkedList; -import java.util.List; -import java.util.Map; -import java.util.Set; -import java.util.function.Supplier; -import java.util.stream.Collectors; - -import static org.labkey.api.data.ColumnRenderPropertiesImpl.STORAGE_UNIQUE_ID_CONCEPT_URI; -import static org.labkey.api.data.ColumnRenderPropertiesImpl.TEXT_CHOICE_CONCEPT_URI; -import static org.labkey.api.exp.api.ExperimentService.MODULE_NAME; -import static org.labkey.api.exp.query.ExpSchema.SAMPLE_FILES_TABLE; - -public class ExperimentModule extends SpringModule -{ - private static final String SAMPLE_TYPE_WEB_PART_NAME = "Sample Types"; - private static final String PROTOCOL_WEB_PART_NAME = "Protocols"; - - public static final String AMOUNT_AND_UNIT_UPGRADE_PROP = "AmountAndUnitAudit"; - public static final String TRANSACTION_ID_PROP = "AuditTransactionId"; - public static final String AUDIT_COUNT_PROP = "AuditRecordCount"; - public static final String EXPERIMENT_RUN_WEB_PART_NAME = "Experiment Runs"; - - @Override - public String getName() - { - return MODULE_NAME; - } - - @Override - public Double getSchemaVersion() - { - return 25.013; - } - - @Nullable - @Override - public UpgradeCode getUpgradeCode() - { - return new ExperimentUpgradeCode(); - } - - @Override - protected void init() - { - addController("experiment", ExperimentController.class); - addController("experiment-types", TypesController.class); - addController("property", PropertyController.class); - ExperimentService.setInstance(new ExperimentServiceImpl()); - SampleTypeService.setInstance(new SampleTypeServiceImpl()); - DefaultValueService.setInstance(new DefaultValueServiceImpl()); - StorageProvisioner.setInstance(StorageProvisionerImpl.get()); - ExpLineageService.setInstance(new ExpLineageServiceImpl()); - - PropertyServiceImpl propertyServiceImpl = new PropertyServiceImpl(); - PropertyService.setInstance(propertyServiceImpl); - UsageMetricsService.get().registerUsageMetrics(getName(), propertyServiceImpl); - - UsageMetricsService.get().registerUsageMetrics(getName(), FileLinkMetricsProvider.getInstance()); - - ExperimentProperty.register(); - SamplesSchema.register(this); - ExpSchema.register(this); - - PropertyService.get().registerDomainKind(new SampleTypeDomainKind()); - PropertyService.get().registerDomainKind(new DataClassDomainKind()); - PropertyService.get().registerDomainKind(new VocabularyDomainKind()); - - QueryService.get().addCompareType(new ChildOfCompareType()); - QueryService.get().addCompareType(new ParentOfCompareType()); - QueryService.get().addCompareType(new LineageCompareType()); - QueryService.get().registerMethod(ChildOfMethod.NAME, new ChildOfMethod(), JdbcType.BOOLEAN, 2, 3); - QueryService.get().registerMethod(ParentOfMethod.NAME, new ParentOfMethod(), JdbcType.BOOLEAN, 2, 3); - QueryService.get().addQueryListener(new ExperimentQueryChangeListener()); - QueryService.get().addQueryListener(new PropertyQueryChangeListener()); - - PropertyService.get().registerValidatorKind(new RegExValidator()); - PropertyService.get().registerValidatorKind(new RangeValidator()); - PropertyService.get().registerValidatorKind(new LookupValidator()); - PropertyService.get().registerValidatorKind(new LengthValidator()); - PropertyService.get().registerValidatorKind(new TextChoiceValidator()); - - ExperimentService.get().registerExperimentDataHandler(new DefaultExperimentDataHandler()); - ExperimentService.get().registerProtocolInputCriteria(new FilterProtocolInputCriteria.Factory()); - ExperimentService.get().registerNameExpressionType("sampletype", "exp", "MaterialSource", "nameexpression"); - ExperimentService.get().registerNameExpressionType("aliquots", "exp", "MaterialSource", "aliquotnameexpression"); - ExperimentService.get().registerNameExpressionType("dataclass", "exp", "DataClass", "nameexpression"); - - OptionalFeatureService.get().addExperimentalFeatureFlag(AppProps.EXPERIMENTAL_RESOLVE_PROPERTY_URI_COLUMNS, "Resolve property URIs as columns on experiment tables", - "If a column is not found on an experiment table, attempt to resolve the column name as a Property URI and add it as a property column", false); - if (CoreSchema.getInstance().getSqlDialect().isSqlServer()) - { - OptionalFeatureService.get().addExperimentalFeatureFlag(NameGenerator.EXPERIMENTAL_WITH_COUNTER, "Use strict incremental withCounter and rootSampleCount expression", - "When withCounter or rootSampleCount is used in name expression, make sure the count increments one-by-one and does not jump.", true); - } - else - { - OptionalFeatureService.get().addExperimentalFeatureFlag(SAMPLE_FILES_TABLE, "Manage Stale Sample Files", - "Allow 'Manage Sample Files' to view sample files that are no longer referenced by samples", false); - - OptionalFeatureService.get().addExperimentalFeatureFlag(NameGenerator.EXPERIMENTAL_ALLOW_GAP_COUNTER, "Allow gap with withCounter and rootSampleCount expression", - "Check this option if gaps in the count generated by withCounter or rootSampleCount name expression are allowed.", true); - } - OptionalFeatureService.get().addExperimentalFeatureFlag(AppProps.QUANTITY_COLUMN_SUFFIX_TESTING, "Quantity column suffix testing", - "If a column name contains a \"__\" suffix, this feature allows for testing it as a Quantity display column", false); - - RoleManager.registerPermission(new DesignVocabularyPermission(), true); - RoleManager.registerRole(new SampleTypeDesignerRole()); - RoleManager.registerRole(new DataClassDesignerRole()); - - AttachmentService.get().registerAttachmentType(ExpRunAttachmentType.get()); - AttachmentService.get().registerAttachmentType(ExpProtocolAttachmentType.get()); - - WebdavService.get().addExpDataProvider((path, container) -> ExperimentService.get().getAllExpDataByURL(path, container)); - ExperimentService.get().registerObjectReferencer(ExperimentServiceImpl.get()); - - addModuleProperty(new LineageMaximumDepthModuleProperty(this)); - WarningService.get().register(new ExperimentWarningProvider()); - } - - @Override - public boolean hasScripts() - { - return true; - } - - @Override - @NotNull - protected Collection createWebPartFactories() - { - List result = new ArrayList<>(); - - BaseWebPartFactory runGroupsFactory = new BaseWebPartFactory(RunGroupWebPart.WEB_PART_NAME, WebPartFactory.LOCATION_BODY, WebPartFactory.LOCATION_RIGHT) - { - @Override - public WebPartView getWebPartView(@NotNull ViewContext portalCtx, @NotNull Portal.WebPart webPart) - { - return new RunGroupWebPart(portalCtx, WebPartFactory.LOCATION_RIGHT.equalsIgnoreCase(webPart.getLocation()), webPart); - } - }; - runGroupsFactory.addLegacyNames("Experiments", "Experiment", "Experiment Navigator", "Narrow Experiments"); - result.add(runGroupsFactory); - - BaseWebPartFactory runTypesFactory = new BaseWebPartFactory(RunTypeWebPart.WEB_PART_NAME, WebPartFactory.LOCATION_BODY, WebPartFactory.LOCATION_RIGHT) - { - @Override - public WebPartView getWebPartView(@NotNull ViewContext portalCtx, @NotNull Portal.WebPart webPart) - { - return new RunTypeWebPart(); - } - }; - result.add(runTypesFactory); - - result.add(new ExperimentRunWebPartFactory()); - BaseWebPartFactory sampleTypeFactory = new BaseWebPartFactory(SAMPLE_TYPE_WEB_PART_NAME, WebPartFactory.LOCATION_BODY, WebPartFactory.LOCATION_RIGHT) - { - @Override - public WebPartView getWebPartView(@NotNull ViewContext portalCtx, @NotNull Portal.WebPart webPart) - { - return new SampleTypeWebPart(WebPartFactory.LOCATION_RIGHT.equalsIgnoreCase(webPart.getLocation()), portalCtx); - } - }; - sampleTypeFactory.addLegacyNames("Narrow Sample Sets", "Sample Sets"); - result.add(sampleTypeFactory); - result.add(new AlwaysAvailableWebPartFactory("Samples Menu", false, false, WebPartFactory.LOCATION_MENUBAR) { - @Override - public WebPartView getWebPartView(@NotNull ViewContext portalCtx, @NotNull Portal.WebPart webPart) - { - WebPartView view = new JspView<>("/org/labkey/experiment/samplesAndAnalytes.jsp", webPart); - view.setTitle("Samples"); - return view; - } - }); - - result.add(new AlwaysAvailableWebPartFactory("Data Classes", false, false, WebPartFactory.LOCATION_BODY, WebPartFactory.LOCATION_RIGHT) { - @Override - public WebPartView getWebPartView(@NotNull ViewContext portalCtx, @NotNull Portal.WebPart webPart) - { - return new DataClassWebPart(WebPartFactory.LOCATION_RIGHT.equalsIgnoreCase(webPart.getLocation()), portalCtx, webPart); - } - }); - - BaseWebPartFactory narrowProtocolFactory = new BaseWebPartFactory(PROTOCOL_WEB_PART_NAME, WebPartFactory.LOCATION_RIGHT) - { - @Override - public WebPartView getWebPartView(@NotNull ViewContext portalCtx, @NotNull Portal.WebPart webPart) - { - return new ProtocolWebPart(WebPartFactory.LOCATION_RIGHT.equalsIgnoreCase(webPart.getLocation()), portalCtx); - } - }; - narrowProtocolFactory.addLegacyNames("Narrow Protocols"); - result.add(narrowProtocolFactory); - - return result; - } - - private void addDataResourceResolver(String categoryName) - { - SearchService.get().addResourceResolver(categoryName, new SearchService.ResourceResolver() - { - @Override - public WebdavResource resolve(@NotNull String resourceIdentifier) - { - ExpDataImpl data = ExpDataImpl.fromDocumentId(resourceIdentifier); - if (data == null) - return null; - - return data.createIndexDocument(null); - } - - @Override - public Map getCustomSearchJson(User user, @NotNull String resourceIdentifier) - { - ExpDataImpl data = ExpDataImpl.fromDocumentId(resourceIdentifier); - if (data == null) - return null; - - return ExperimentJSONConverter.serializeData(data, user, ExperimentJSONConverter.DEFAULT_SETTINGS).toMap(); - } - - @Override - public Map> getCustomSearchJsonMap(User user, @NotNull Collection resourceIdentifiers) - { - Map idDataMap = ExpDataImpl.fromDocumentIds(resourceIdentifiers); - if (idDataMap == null) - return null; - - Map> searchJsonMap = new HashMap<>(); - for (String resourceIdentifier : idDataMap.keySet()) - searchJsonMap.put(resourceIdentifier, ExperimentJSONConverter.serializeData(idDataMap.get(resourceIdentifier), user, ExperimentJSONConverter.DEFAULT_SETTINGS).toMap()); - return searchJsonMap; - } - }); - } - - private void addDataClassResourceResolver(String categoryName) - { - SearchService.get().addResourceResolver(categoryName, new SearchService.ResourceResolver(){ - @Override - public Map getCustomSearchJson(User user, @NotNull String resourceIdentifier) - { - int rowId = NumberUtils.toInt(resourceIdentifier.replace(categoryName + ":", "")); - if (rowId == 0) - return null; - - ExpDataClass dataClass = ExperimentService.get().getDataClass(rowId); - if (dataClass == null) - return null; - - Map properties = ExperimentJSONConverter.serializeExpObject(dataClass, null, ExperimentJSONConverter.DEFAULT_SETTINGS, user).toMap(); - - //Need to map to proper Icon - properties.put("type", "dataClass" + (dataClass.getCategory() != null ? ":" + dataClass.getCategory() : "")); - - return properties; - } - }); - } - - private void addSampleTypeResourceResolver(String categoryName) - { - SearchService.get().addResourceResolver(categoryName, new SearchService.ResourceResolver(){ - @Override - public Map getCustomSearchJson(User user, @NotNull String resourceIdentifier) - { - int rowId = NumberUtils.toInt(resourceIdentifier.replace(categoryName + ":", "")); - if (rowId == 0) - return null; - - ExpSampleType sampleType = SampleTypeService.get().getSampleType(rowId); - if (sampleType == null) - return null; - - Map properties = ExperimentJSONConverter.serializeExpObject(sampleType, null, ExperimentJSONConverter.DEFAULT_SETTINGS, user).toMap(); - - //Need to map to proper Icon - properties.put("type", "sampleSet"); - - return properties; - } - }); - } - - private void addSampleResourceResolver(String categoryName) - { - SearchService.get().addResourceResolver(categoryName, new SearchService.ResourceResolver(){ - @Override - public Map getCustomSearchJson(User user, @NotNull String resourceIdentifier) - { - int rowId = NumberUtils.toInt(resourceIdentifier.replace(categoryName + ":", "")); - if (rowId == 0) - return null; - - ExpMaterial material = ExperimentService.get().getExpMaterial(rowId); - if (material == null) - return null; - - return ExperimentJSONConverter.serializeMaterial(material, user, ExperimentJSONConverter.DEFAULT_SETTINGS).toMap(); - } - - @Override - public Map> getCustomSearchJsonMap(User user, @NotNull Collection resourceIdentifiers) - { - Set rowIds = new HashSet<>(); - Map rowIdIdentifierMap = new LongHashMap<>(); - for (String resourceIdentifier : resourceIdentifiers) - { - long rowId = NumberUtils.toLong(resourceIdentifier.replace(categoryName + ":", "")); - if (rowId != 0) - { - rowIds.add(rowId); - rowIdIdentifierMap.put(rowId, resourceIdentifier); - } - } - - Map> searchJsonMap = new HashMap<>(); - for (ExpMaterial material : ExperimentService.get().getExpMaterials(rowIds)) - { - searchJsonMap.put( - rowIdIdentifierMap.get(material.getRowId()), - ExperimentJSONConverter.serializeMaterial(material, user, ExperimentJSONConverter.DEFAULT_SETTINGS).toMap() - ); - } - - return searchJsonMap; - } - }); - } - - @Override - protected void startupAfterSpringConfig(ModuleContext moduleContext) - { - SearchService ss = SearchService.get(); -// ss.addSearchCategory(OntologyManager.conceptCategory); - ss.addSearchCategory(ExpSampleTypeImpl.searchCategory); - ss.addSearchCategory(ExpSampleTypeImpl.mediaSearchCategory); - ss.addSearchCategory(ExpMaterialImpl.searchCategory); - ss.addSearchCategory(ExpMaterialImpl.mediaSearchCategory); - ss.addSearchCategory(ExpDataClassImpl.SEARCH_CATEGORY); - ss.addSearchCategory(ExpDataClassImpl.MEDIA_SEARCH_CATEGORY); - ss.addSearchCategory(ExpDataImpl.expDataCategory); - ss.addSearchCategory(ExpDataImpl.expMediaDataCategory); - ss.addSearchResultTemplate(new ExpDataImpl.DataSearchResultTemplate()); - addDataResourceResolver(ExpDataImpl.expDataCategory.getName()); - addDataResourceResolver(ExpDataImpl.expMediaDataCategory.getName()); - addDataClassResourceResolver(ExpDataClassImpl.SEARCH_CATEGORY.getName()); - addDataClassResourceResolver(ExpDataClassImpl.MEDIA_SEARCH_CATEGORY.getName()); - addSampleTypeResourceResolver(ExpSampleTypeImpl.searchCategory.getName()); - addSampleTypeResourceResolver(ExpSampleTypeImpl.mediaSearchCategory.getName()); - addSampleResourceResolver(ExpMaterialImpl.searchCategory.getName()); - addSampleResourceResolver(ExpMaterialImpl.mediaSearchCategory.getName()); - ss.addDocumentProvider(ExperimentServiceImpl.get()); - - PipelineService.get().registerPipelineProvider(new ExperimentPipelineProvider(this)); - ExperimentService.get().registerExperimentRunTypeSource(container -> Collections.singleton(ExperimentRunType.ALL_RUNS_TYPE)); - ExperimentService.get().registerDataType(new LogDataType()); - - AuditLogService.get().registerAuditType(new DomainAuditProvider()); - AuditLogService.get().registerAuditType(new DomainPropertyAuditProvider()); - AuditLogService.get().registerAuditType(new ExperimentAuditProvider()); - AuditLogService.get().registerAuditType(new SampleTypeAuditProvider()); - AuditLogService.get().registerAuditType(new SampleTimelineAuditProvider()); - - FileContentService fileContentService = FileContentService.get(); - if (null != fileContentService) - { - fileContentService.addFileListener(new ExpDataFileListener()); - fileContentService.addFileListener(new TableUpdaterFileListener(ExperimentService.get().getTinfoExperimentRun(), "FilePathRoot", TableUpdaterFileListener.Type.fileRootPath, "RowId")); - fileContentService.addFileListener(new FileLinkFileListener()); - } - ContainerManager.addContainerListener(new ContainerManager.ContainerListener() - { - @Override - public void containerDeleted(Container c, User user) - { - try - { - ExperimentService.get().deleteAllExpObjInContainer(c, user); - } - catch (ExperimentException ee) - { - throw new RuntimeException(ee); - } - } - }, - // This is in the Last group because when a container is deleted, - // the Experiment listener needs to be called after the Study listener, - // because Study needs the metadata held by Experiment to delete properly. - // but it should be before the CoreContainerListener - ContainerManager.ContainerListener.Order.Last); - - if (ModuleLoader.getInstance().shouldInsertData()) - SystemProperty.registerProperties(); - - FolderSerializationRegistry folderRegistry = FolderSerializationRegistry.get(); - if (null != folderRegistry) - { - folderRegistry.addFactories(new FolderXarWriterFactory(), new FolderXarImporterFactory()); - folderRegistry.addWriterFactory(new SampleTypeFolderWriter.SampleTypeDesignWriter.Factory()); - folderRegistry.addWriterFactory(new SampleTypeFolderWriter.SampleTypeDataWriter.Factory()); - folderRegistry.addWriterFactory(new DataClassFolderWriter.DataClassDesignWriter.Factory()); - folderRegistry.addWriterFactory(new DataClassFolderWriter.DataClassDataWriter.Factory()); - folderRegistry.addImportFactory(new SampleTypeFolderImporter.Factory()); - folderRegistry.addImportFactory(new DataClassFolderImporter.Factory()); - folderRegistry.addImportFactory(new SampleStatusFolderImporter.Factory()); - } - - AttachmentService.get().registerAttachmentType(ExpDataClassType.get()); - - WebdavService.get().addProvider(new ScriptsResourceProvider()); - - SystemMaintenance.addTask(new FileLinkMetricsMaintenanceTask()); - - UsageMetricsService svc = UsageMetricsService.get(); - if (null != svc) - { - svc.registerUsageMetrics(getName(), () -> { - Map results = new HashMap<>(); - - DbSchema schema = ExperimentService.get().getSchema(); - if (AssayService.get() != null) - { - Map assayMetrics = new HashMap<>(); - SQLFragment baseRunSQL = new SQLFragment("SELECT COUNT(*) FROM ").append(ExperimentService.get().getTinfoExperimentRun(), "r").append(" WHERE lsid LIKE ?"); - SQLFragment baseProtocolSQL = new SQLFragment("SELECT * FROM ").append(ExperimentService.get().getTinfoProtocol(), "p").append(" WHERE lsid LIKE ? AND ApplicationType = ?"); - for (AssayProvider assayProvider : AssayService.get().getAssayProviders()) - { - Map protocolMetrics = new HashMap<>(); - - // Run count across all assay designs of this type - SQLFragment runSQL = new SQLFragment(baseRunSQL); - runSQL.add(Lsid.namespaceLikeString(assayProvider.getRunLSIDPrefix())); - protocolMetrics.put("runCount", new SqlSelector(schema, runSQL).getObject(Long.class)); - - // Number of assay designs of this type - SQLFragment protocolSQL = new SQLFragment(baseProtocolSQL); - protocolSQL.add(assayProvider.getProtocolPattern()); - protocolSQL.add(ExpProtocol.ApplicationType.ExperimentRun.toString()); - List protocols = new SqlSelector(schema, protocolSQL).getArrayList(Protocol.class); - protocolMetrics.put("protocolCount", protocols.size()); - - List wrappedProtocols = protocols.stream().map(ExpProtocolImpl::new).collect(Collectors.toList()); - - protocolMetrics.put("resultRowCount", assayProvider.getResultRowCount(wrappedProtocols)); - - // Primary implementation class - protocolMetrics.put("implementingClass", assayProvider.getClass()); - - assayMetrics.put(assayProvider.getName(), protocolMetrics); - } - assayMetrics.put("autoLinkedAssayCount", new SqlSelector(schema, "SELECT COUNT(*) FROM exp.protocol EP JOIN exp.objectPropertiesView OP ON EP.lsid = OP.objecturi WHERE OP.propertyuri = 'terms.labkey.org#AutoCopyTargetContainer'").getObject(Long.class)); - assayMetrics.put("protocolsWithTransformScriptCount", new SqlSelector(schema, "SELECT COUNT(*) FROM exp.protocol EP JOIN exp.objectPropertiesView OP ON EP.lsid = OP.objecturi WHERE OP.name = 'TransformScript' AND status = 'Active'").getObject(Long.class)); - assayMetrics.put("protocolsWithTransformScriptRunOnEditCount", new SqlSelector(schema, "SELECT COUNT(*) FROM exp.protocol EP JOIN exp.objectPropertiesView OP ON EP.lsid = OP.objecturi WHERE OP.name = 'TransformScript' AND status = 'Active' AND OP.stringvalue LIKE '%\"INSERT\"%'").getObject(Long.class)); - assayMetrics.put("protocolsWithTransformScriptRunOnImportCount", new SqlSelector(schema, "SELECT COUNT(*) FROM exp.protocol EP JOIN exp.objectPropertiesView OP ON EP.lsid = OP.objecturi WHERE OP.name = 'TransformScript' AND status = 'Active' AND OP.stringvalue LIKE '%\"INSERT\"%'").getObject(Long.class)); - - assayMetrics.put("standardAssayWithPlateSupportCount", new SqlSelector(schema, "SELECT COUNT(*) FROM exp.protocol EP JOIN exp.objectPropertiesView OP ON EP.lsid = OP.objecturi WHERE OP.name = 'PlateMetadata' AND floatValue = 1").getObject(Long.class)); - SQLFragment runsWithPlateSQL = new SQLFragment(""" - SELECT COUNT(*) FROM exp.experimentrun r - INNER JOIN exp.object o ON o.objectUri = r.lsid - INNER JOIN exp.objectproperty op ON op.objectId = o.objectId - WHERE op.propertyid IN ( - SELECT propertyid FROM exp.propertydescriptor WHERE name = ? AND lookupquery = ? - )"""); - assayMetrics.put("standardAssayRunsWithPlateTemplate", new SqlSelector(schema, new SQLFragment(runsWithPlateSQL).add("PlateTemplate").add("PlateTemplate")).getObject(Long.class)); - assayMetrics.put("standardAssayRunsWithPlateSet", new SqlSelector(schema, new SQLFragment(runsWithPlateSQL).add("PlateSet").add("PlateSet")).getObject(Long.class)); - - assayMetrics.put("assayRunsFileColumnCount", new SqlSelector(schema, """ - SELECT COUNT(DISTINCT DD.DomainURI) FROM - exp.PropertyDescriptor D\s - JOIN exp.PropertyDomain PD ON D.propertyId = PD.propertyid - JOIN exp.DomainDescriptor DD on PD.domainID = DD.domainId - WHERE DD.domainUri LIKE ? AND D.rangeURI = ?""", "urn:lsid:%:" + ExpProtocol.AssayDomainTypes.Run.getPrefix() + ".%", PropertyType.FILE_LINK.getTypeUri()).getObject(Long.class)); - - assayMetrics.put("assayResultsFileColumnCount", new SqlSelector(schema, """ - SELECT COUNT(DISTINCT DD.DomainURI) FROM - exp.PropertyDescriptor D\s - JOIN exp.PropertyDomain PD ON D.propertyId = PD.propertyid - JOIN exp.DomainDescriptor DD on PD.domainID = DD.domainId - WHERE DD.domainUri LIKE ? AND D.rangeURI = ?""", "urn:lsid:%:" + ExpProtocol.AssayDomainTypes.Result.getPrefix() + ".%", PropertyType.FILE_LINK.getTypeUri()).getObject(Long.class)); - - Map sampleLookupCountMetrics = new HashMap<>(); - SQLFragment baseAssaySampleLookupSQL = new SQLFragment("SELECT COUNT(*) FROM exp.propertydescriptor WHERE (lookupschema = 'samples' OR (lookupschema = 'exp' AND lookupquery = 'Materials')) AND propertyuri LIKE ?"); - - SQLFragment batchAssaySampleLookupSQL = new SQLFragment(baseAssaySampleLookupSQL); - batchAssaySampleLookupSQL.add("urn:lsid:%:" + ExpProtocol.AssayDomainTypes.Batch.getPrefix() + ".%"); - sampleLookupCountMetrics.put("batchDomain", new SqlSelector(schema, batchAssaySampleLookupSQL).getObject(Long.class)); - - SQLFragment runAssaySampleLookupSQL = new SQLFragment(baseAssaySampleLookupSQL); - runAssaySampleLookupSQL.add("urn:lsid:%:" + ExpProtocol.AssayDomainTypes.Run.getPrefix() + ".%"); - sampleLookupCountMetrics.put("runDomain", new SqlSelector(schema, runAssaySampleLookupSQL).getObject(Long.class)); - - SQLFragment resultAssaySampleLookupSQL = new SQLFragment(baseAssaySampleLookupSQL); - resultAssaySampleLookupSQL.add("urn:lsid:%:" + ExpProtocol.AssayDomainTypes.Result.getPrefix() + ".%"); - sampleLookupCountMetrics.put("resultDomain", new SqlSelector(schema, resultAssaySampleLookupSQL).getObject(Long.class)); - - SQLFragment resultAssayMultipleSampleLookupSQL = new SQLFragment( - """ - SELECT COUNT(*) FROM ( - SELECT PD.domainid, COUNT(*) AS PropCount - FROM exp.propertydescriptor D - JOIN exp.PropertyDomain PD ON D.propertyId = PD.propertyid - WHERE (lookupschema = 'samples' OR (lookupschema = 'exp' AND lookupquery = 'Materials')) - AND propertyuri LIKE ? - GROUP BY PD.domainid - ) X WHERE X.PropCount > 1""" - ); - resultAssayMultipleSampleLookupSQL.add("urn:lsid:%:" + ExpProtocol.AssayDomainTypes.Result.getPrefix() + ".%"); - sampleLookupCountMetrics.put("resultDomainWithMultiple", new SqlSelector(schema, resultAssayMultipleSampleLookupSQL).getObject(Long.class)); - - assayMetrics.put("sampleLookupCount", sampleLookupCountMetrics); - - - // Putting these metrics at the same level as the other BooleanColumnCount metrics (e.g., sampleTypeWithBooleanColumnCount) - results.put("assayResultWithBooleanColumnCount", new SqlSelector(schema, """ - SELECT COUNT(DISTINCT DD.DomainURI) FROM - exp.PropertyDescriptor D\s - JOIN exp.PropertyDomain PD ON D.propertyId = PD.propertyid - JOIN exp.DomainDescriptor DD on PD.domainID = DD.domainId - WHERE D.propertyURI LIKE ? AND D.rangeURI = ?""", "urn:lsid:%:" + ExpProtocol.AssayDomainTypes.Result.getPrefix() + ".%", PropertyType.BOOLEAN.getTypeUri()).getObject(Long.class)); - - results.put("assayRunWithBooleanColumnCount", new SqlSelector(schema, """ - SELECT COUNT(DISTINCT DD.DomainURI) FROM - exp.PropertyDescriptor D\s - JOIN exp.PropertyDomain PD ON D.propertyId = PD.propertyid - JOIN exp.DomainDescriptor DD on PD.domainID = DD.domainId - WHERE D.propertyURI LIKE ? AND D.rangeURI = ?""", "urn:lsid:%:" + ExpProtocol.AssayDomainTypes.Run.getPrefix() + ".%", PropertyType.BOOLEAN.getTypeUri()).getObject(Long.class)); - - results.put("assay", assayMetrics); - } - - results.put("autoLinkedSampleSetCount", new SqlSelector(schema, "SELECT COUNT(*) FROM exp.materialsource WHERE autoLinkTargetContainer IS NOT NULL").getObject(Long.class)); - results.put("sampleSetCount", new SqlSelector(schema, "SELECT COUNT(*) FROM exp.materialsource").getObject(Long.class)); - - if (schema.getSqlDialect().isPostgreSQL()) // SQLServer does not support regular expression queries - { - Collection> numSampleCounts = new SqlSelector(schema, """ - SELECT totalCount, numberNameCount FROM - (SELECT cpastype, COUNT(*) AS totalCount from exp.material GROUP BY cpastype) t - JOIN - (SELECT cpastype, COUNT(*) AS numberNameCount FROM exp.material m WHERE m.name SIMILAR TO '[0-9.]*' GROUP BY cpastype) ns - ON t.cpastype = ns.cpastype""").getMapCollection(); - results.put("sampleSetWithNumberNamesCount", numSampleCounts.size()); - results.put("sampleSetWithOnlyNumberNamesCount", numSampleCounts.stream().filter( - map -> (Long) map.get("totalCount") > 0 && map.get("totalCount") == map.get("numberNameCount") - ).count()); - } - UserSchema userSchema = AuditLogService.getAuditLogSchema(User.getSearchUser(), ContainerManager.getRoot()); - FilteredTable table = (FilteredTable) userSchema.getTable(SampleTimelineAuditEvent.EVENT_TYPE); - - SQLFragment sql = new SQLFragment("SELECT COUNT(*)\n" + - " FROM (\n" + - " -- updates that are marked as lineage updates\n" + - " (SELECT DISTINCT transactionId\n" + - " FROM " + table.getRealTable().getFromSQL("").getSQL() +"\n" + - " WHERE islineageupdate = " + schema.getSqlDialect().getBooleanTRUE() + "\n" + - " AND comment = 'Sample was updated.'\n" + - " ) a1\n" + - " JOIN\n" + - " -- but have associated entries that are not lineage updates\n" + - " (SELECT DISTINCT transactionid\n" + - " FROM " + table.getRealTable().getFromSQL("").getSQL() + "\n" + - " WHERE islineageupdate = " + schema.getSqlDialect().getBooleanFALSE() + ") a2\n" + - " ON a1.transactionid = a2.transactionid\n" + - " )"); - - results.put("sampleLineageAuditDiscrepancyCount", new SqlSelector(schema, sql.getSQL()).getObject(Long.class)); - - results.put("sampleCount", new SqlSelector(schema, "SELECT COUNT(*) FROM exp.material").getObject(Long.class)); - results.put("aliquotCount", new SqlSelector(schema, "SELECT COUNT(*) FROM exp.material where aliquotedfromlsid IS NOT NULL").getObject(Long.class)); - results.put("sampleNullAmountCount", new SqlSelector(schema, "SELECT COUNT(*) FROM exp.material WHERE storedamount IS NULL").getObject(Long.class)); - results.put("sampleNegativeAmountCount", new SqlSelector(schema, "SELECT COUNT(*) FROM exp.material WHERE storedamount < 0").getObject(Long.class)); - results.put("sampleUnitsDifferCount", new SqlSelector(schema, "SELECT COUNT(*) from exp.material m JOIN exp.materialSource s ON m.materialsourceid = s.rowid WHERE m.units != s.metricunit").getObject(Long.class)); - results.put("sampleTypesWithoutUnitsCount", new SqlSelector(schema, "SELECT COUNT(*) from exp.materialSource WHERE category IS NULL AND metricunit IS NULL").getObject(Long.class)); - - results.put("duplicateSampleMaterialNameCount", new SqlSelector(schema, "SELECT COUNT(*) as duplicateCount FROM " + - "(SELECT name, cpastype FROM exp.material WHERE cpastype <> 'Material' GROUP BY name, cpastype HAVING COUNT(*) > 1) d").getObject(Long.class)); - results.put("duplicateSpecimenMaterialNameCount", new SqlSelector(schema, "SELECT COUNT(*) as duplicateCount FROM " + - "(SELECT name, cpastype FROM exp.material WHERE cpastype = 'Material' GROUP BY name, cpastype HAVING COUNT(*) > 1) d").getObject(Long.class)); - String duplicateCaseInsensitiveSampleNameCountSql = """ - SELECT COUNT(*) FROM - ( - SELECT 1 AS found - FROM exp.material - WHERE materialsourceid IS NOT NULL - GROUP BY LOWER(name), materialsourceid - HAVING COUNT(*) > 1 - ) AS duplicates - """; - String duplicateCaseInsensitiveDataNameCountSql = """ - SELECT COUNT(*) FROM - ( - SELECT 1 AS found - FROM exp.data - WHERE classid IS NOT NULL - GROUP BY LOWER(name), classid - HAVING COUNT(*) > 1 - ) AS duplicates - """; - results.put("duplicateCaseInsensitiveSampleNameCount", new SqlSelector(schema, duplicateCaseInsensitiveSampleNameCountSql).getObject(Long.class)); - results.put("duplicateCaseInsensitiveDataNameCount", new SqlSelector(schema, duplicateCaseInsensitiveDataNameCountSql).getObject(Long.class)); - - results.put("dataClassCount", new SqlSelector(schema, "SELECT COUNT(*) FROM exp.dataclass").getObject(Long.class)); - results.put("dataClassRowCount", new SqlSelector(schema, "SELECT COUNT(*) FROM exp.data WHERE classid IN (SELECT rowid FROM exp.dataclass)").getObject(Long.class)); - results.put("dataWithDataParentsCount", new SqlSelector(schema, "SELECT COUNT(DISTINCT d.sourceApplicationId) FROM exp.data d\n" + - "JOIN exp.datainput di ON di.targetapplicationid = d.sourceapplicationid").getObject(Long.class)); - if (schema.getSqlDialect().isPostgreSQL()) - { - Collection> numDataClassObjectsCounts = new SqlSelector(schema, """ - SELECT totalCount, numberNameCount FROM - (SELECT cpastype, COUNT(*) AS totalCount from exp.data GROUP BY cpastype) t - JOIN - (SELECT cpastype, COUNT(*) AS numberNameCount FROM exp.data m WHERE m.name SIMILAR TO '[0-9.]*' GROUP BY cpastype) ns - ON t.cpastype = ns.cpastype""").getMapCollection(); - results.put("dataClassWithNumberNamesCount", numDataClassObjectsCounts.size()); - results.put("dataClassWithOnlyNumberNamesCount", numDataClassObjectsCounts.stream().filter(map -> - (Long) map.get("totalCount") > 0 && map.get("totalCount") == map.get("numberNameCount")).count()); - } - - results.put("ontologyPrincipalConceptCodeCount", new SqlSelector(schema, "SELECT COUNT(*) FROM exp.propertydescriptor WHERE principalconceptcode IS NOT NULL").getObject(Long.class)); - results.put("ontologyLookupColumnCount", new SqlSelector(schema, "SELECT COUNT(*) FROM exp.propertydescriptor WHERE concepturi = ?", OntologyService.conceptCodeConceptURI).getObject(Long.class)); - results.put("ontologyConceptSubtreeCount", new SqlSelector(schema, "SELECT COUNT(*) FROM exp.propertydescriptor WHERE conceptsubtree IS NOT NULL").getObject(Long.class)); - results.put("ontologyConceptImportColumnCount", new SqlSelector(schema, "SELECT COUNT(*) FROM exp.propertydescriptor WHERE conceptimportcolumn IS NOT NULL").getObject(Long.class)); - results.put("ontologyConceptLabelColumnCount", new SqlSelector(schema, "SELECT COUNT(*) FROM exp.propertydescriptor WHERE conceptlabelcolumn IS NOT NULL").getObject(Long.class)); - - results.put("scannableColumnCount", new SqlSelector(schema, "SELECT COUNT(*) FROM exp.propertydescriptor WHERE scannable = ?", true).getObject(Long.class)); - results.put("uniqueIdColumnCount", new SqlSelector(schema, "SELECT COUNT(*) FROM exp.propertydescriptor WHERE concepturi = ?", STORAGE_UNIQUE_ID_CONCEPT_URI).getObject(Long.class)); - results.put("sampleTypeWithUniqueIdCount", new SqlSelector(schema, """ - SELECT COUNT(DISTINCT DD.DomainURI) FROM - exp.PropertyDescriptor D\s - JOIN exp.PropertyDomain PD ON D.propertyId = PD.propertyid - JOIN exp.DomainDescriptor DD on PD.domainID = DD.domainId - WHERE D.conceptURI = ?""", STORAGE_UNIQUE_ID_CONCEPT_URI).getObject(Long.class)); - - results.put("fileColumnCount", new SqlSelector(schema, "SELECT COUNT(*) FROM exp.propertydescriptor WHERE rangeURI = ?", PropertyType.FILE_LINK.getTypeUri()).getObject(Long.class)); - results.put("sampleTypeWithFileColumnCount", new SqlSelector(schema, """ - SELECT COUNT(DISTINCT DD.DomainURI) FROM - exp.PropertyDescriptor D\s - JOIN exp.PropertyDomain PD ON D.propertyId = PD.propertyid - JOIN exp.DomainDescriptor DD on PD.domainID = DD.domainId - WHERE DD.storageSchemaName = ? AND D.rangeURI = ?""", SampleTypeDomainKind.PROVISIONED_SCHEMA_NAME, PropertyType.FILE_LINK.getTypeUri()).getObject(Long.class)); - results.put("sampleTypeWithBooleanColumnCount", new SqlSelector(schema, """ - SELECT COUNT(DISTINCT DD.DomainURI) FROM - exp.PropertyDescriptor D\s - JOIN exp.PropertyDomain PD ON D.propertyId = PD.propertyid - JOIN exp.DomainDescriptor DD on PD.domainID = DD.domainId - WHERE DD.storageSchemaName = ? AND D.rangeURI = ?""", SampleTypeDomainKind.PROVISIONED_SCHEMA_NAME, PropertyType.BOOLEAN.getTypeUri()).getObject(Long.class)); - - results.put("sampleTypeAliquotSpecificField", new SqlSelector(schema, """ - SELECT COUNT(DISTINCT D.PropertyURI) FROM - exp.PropertyDescriptor D\s - JOIN exp.PropertyDomain PD ON D.propertyId = PD.propertyid - JOIN exp.DomainDescriptor DD on PD.domainID = DD.domainId - WHERE DD.storageSchemaName = ? AND D.derivationDataScope = ?""", SampleTypeDomainKind.PROVISIONED_SCHEMA_NAME, ExpSchema.DerivationDataScopeType.ChildOnly.name()).getObject(Long.class)); - results.put("sampleTypeParentOnlyField", new SqlSelector(schema, """ - SELECT COUNT(DISTINCT D.PropertyURI) FROM - exp.PropertyDescriptor D\s - JOIN exp.PropertyDomain PD ON D.propertyId = PD.propertyid - JOIN exp.DomainDescriptor DD on PD.domainID = DD.domainId - WHERE DD.storageSchemaName = ? AND (D.derivationDataScope = ? OR D.derivationDataScope IS NULL)""", SampleTypeDomainKind.PROVISIONED_SCHEMA_NAME, ExpSchema.DerivationDataScopeType.ParentOnly.name()).getObject(Long.class)); - results.put("sampleTypeParentAndAliquotField", new SqlSelector(schema, """ - SELECT COUNT(DISTINCT D.PropertyURI) FROM - exp.PropertyDescriptor D\s - JOIN exp.PropertyDomain PD ON D.propertyId = PD.propertyid - JOIN exp.DomainDescriptor DD on PD.domainID = DD.domainId - WHERE DD.storageSchemaName = ? AND D.derivationDataScope = ?""", SampleTypeDomainKind.PROVISIONED_SCHEMA_NAME, ExpSchema.DerivationDataScopeType.All.name()).getObject(Long.class)); - - results.put("attachmentColumnCount", new SqlSelector(schema, "SELECT COUNT(*) FROM exp.propertydescriptor WHERE rangeURI = ?", PropertyType.ATTACHMENT.getTypeUri()).getObject(Long.class)); - results.put("dataClassWithAttachmentColumnCount", new SqlSelector(schema, """ - SELECT COUNT(DISTINCT DD.DomainURI) FROM - exp.PropertyDescriptor D\s - JOIN exp.PropertyDomain PD ON D.propertyId = PD.propertyid - JOIN exp.DomainDescriptor DD on PD.domainID = DD.domainId - WHERE DD.storageSchemaName = ? AND D.rangeURI = ?""", DataClassDomainKind.PROVISIONED_SCHEMA_NAME, PropertyType.ATTACHMENT.getTypeUri()).getObject(Long.class)); - results.put("dataClassWithBooleanColumnCount", new SqlSelector(schema, """ - SELECT COUNT(DISTINCT DD.DomainURI) FROM - exp.PropertyDescriptor D\s - JOIN exp.PropertyDomain PD ON D.propertyId = PD.propertyid - JOIN exp.DomainDescriptor DD on PD.domainID = DD.domainId - WHERE DD.storageSchemaName = ? AND D.rangeURI = ?""", DataClassDomainKind.PROVISIONED_SCHEMA_NAME, PropertyType.BOOLEAN.getTypeUri()).getObject(Long.class)); - - results.put("textChoiceColumnCount", new SqlSelector(schema, "SELECT COUNT(*) FROM exp.propertydescriptor WHERE concepturi = ?", TEXT_CHOICE_CONCEPT_URI).getObject(Long.class)); - - results.put("domainsWithDateTimeColumnCount", new SqlSelector(schema, """ - SELECT COUNT(DISTINCT DD.DomainURI) FROM - exp.PropertyDescriptor D\s - JOIN exp.PropertyDomain PD ON D.propertyId = PD.propertyid - JOIN exp.DomainDescriptor DD on PD.domainID = DD.domainId - WHERE D.rangeURI = ?""", PropertyType.DATE_TIME.getTypeUri()).getObject(Long.class)); - - results.put("domainsWithDateColumnCount", new SqlSelector(schema, """ - SELECT COUNT(DISTINCT DD.DomainURI) FROM - exp.PropertyDescriptor D\s - JOIN exp.PropertyDomain PD ON D.propertyId = PD.propertyid - JOIN exp.DomainDescriptor DD on PD.domainID = DD.domainId - WHERE D.rangeURI = ?""", PropertyType.DATE.getTypeUri()).getObject(Long.class)); - - results.put("domainsWithTimeColumnCount", new SqlSelector(schema, """ - SELECT COUNT(DISTINCT DD.DomainURI) FROM - exp.PropertyDescriptor D\s - JOIN exp.PropertyDomain PD ON D.propertyId = PD.propertyid - JOIN exp.DomainDescriptor DD on PD.domainID = DD.domainId - WHERE D.rangeURI = ?""", PropertyType.TIME.getTypeUri()).getObject(Long.class)); - - results.put("maxObjectObjectId", new SqlSelector(schema, "SELECT MAX(ObjectId) FROM exp.Object").getObject(Long.class)); - results.put("maxMaterialRowId", new SqlSelector(schema, "SELECT MAX(RowId) FROM exp.Material").getObject(Long.class)); - - results.putAll(ExperimentService.get().getDomainMetrics()); - - return results; - }); - } - - ExperimentMigrationSchemaHandler handler = new ExperimentMigrationSchemaHandler(); - DatabaseMigrationService.get().registerSchemaHandler(handler); - DatabaseMigrationService.get().registerTableHandler(new MigrationTableHandler() - { - @Override - public TableInfo getTableInfo() - { - return DbSchema.get("premium", DbSchemaType.Bare).getTable("Exclusions"); - } - - @Override - public void adjustFilter(TableInfo sourceTable, SimpleFilter filter, Set containers) - { - // Include experiment runs that were copied - FilterClause includedClause = handler.getIncludedRowIdClause(sourceTable, FieldKey.fromParts("RunId")); - if (includedClause != null) - filter.addClause(includedClause); - } - }); - DatabaseMigrationService.get().registerTableHandler(new MigrationTableHandler() - { - @Override - public TableInfo getTableInfo() - { - return DbSchema.get("premium", DbSchemaType.Bare).getTable("ExclusionMaps"); - } - - @Override - public void adjustFilter(TableInfo sourceTable, SimpleFilter filter, Set containers) - { - // Include experiment runs that were copied - FilterClause includedClause = handler.getIncludedRowIdClause(sourceTable, FieldKey.fromParts("ExclusionId", "RunId")); - if (includedClause != null) - filter.addClause(includedClause); - } - }); - DatabaseMigrationService.get().registerTableHandler(new MigrationTableHandler() - { - @Override - public TableInfo getTableInfo() - { - return DbSchema.get("assayrequest", DbSchemaType.Bare).getTable("RequestRunsJunction"); - } - - @Override - public void adjustFilter(TableInfo sourceTable, SimpleFilter filter, Set containers) - { - // Include experiment runs that were copied - FilterClause includedClause = handler.getIncludedRowIdClause(sourceTable, FieldKey.fromParts("RunId")); - if (includedClause != null) - filter.addClause(includedClause); - } - }); - DatabaseMigrationService.get().registerSchemaHandler(new SampleTypeMigrationSchemaHandler()); - DataClassMigrationSchemaHandler dcHandler = new DataClassMigrationSchemaHandler(); - DatabaseMigrationService.get().registerSchemaHandler(dcHandler); - ExperimentDeleteService.setInstance(dcHandler); - } - - @Override - @NotNull - public Collection getSummary(Container c) - { - Collection list = new LinkedList<>(); - int runGroupCount = ExperimentService.get().getExperiments(c, null, false, true).size(); - if (runGroupCount > 0) - list.add(StringUtilsLabKey.pluralize(runGroupCount, "Run Group")); - - User user = HttpView.currentContext().getUser(); - - Set runTypes = ExperimentService.get().getExperimentRunTypes(c); - for (ExperimentRunType runType : runTypes) - { - if (runType == ExperimentRunType.ALL_RUNS_TYPE) - continue; - - long runCount = runType.getRunCount(user, c); - if (runCount > 0) - list.add(runCount + " runs of type " + runType.getDescription()); - } - - int dataClassCount = ExperimentService.get().getDataClasses(c, null, false).size(); - if (dataClassCount > 0) - list.add(dataClassCount + " Data Class" + (dataClassCount > 1 ? "es" : "")); - - int sampleTypeCount = SampleTypeService.get().getSampleTypes(c, false).size(); - if (sampleTypeCount > 0) - list.add(sampleTypeCount + " Sample Type" + (sampleTypeCount > 1 ? "s" : "")); - - return list; - } - - @Override - public @NotNull ArrayList getDetailedSummary(Container c, User user) - { - ArrayList summaries = new ArrayList<>(); - - // Assay types - long assayTypeCount = AssayService.get().getAssayProtocols(c).stream().filter(p -> p.getContainer().equals(c)).count(); - if (assayTypeCount > 0) - summaries.add(new Summary(assayTypeCount, "Assay Type")); - - // Run count - int runGroupCount = ExperimentService.get().getExperiments(c, user, false, true).size(); - if (runGroupCount > 0) - summaries.add(new Summary(runGroupCount, "Assay run")); - - // Number of Data Classes - List dataClasses = ExperimentService.get().getDataClasses(c, user, false); - int dataClassCount = dataClasses.size(); - if (dataClassCount > 0) - summaries.add(new Summary(dataClassCount, "Data Class")); - - ExpSchema expSchema = new ExpSchema(user, c); - - // Individual Data Class row counts - { - // The table-level container filter is set to ensure data class types are included - // that may not be defined in the target container but may have rows of data in the target container - TableInfo table = ExpSchema.TableType.DataClasses.createTable(expSchema, null, ContainerFilter.Type.CurrentPlusProjectAndShared.create(c, user)); - - // Issue 47919: The "DataCount" column is filtered to only count data in the target container - if (table instanceof ExpDataClassTableImpl tableImpl) - tableImpl.setDataCountContainerFilter(ContainerFilter.Type.Current.create(c, user)); - - Set columns = new LinkedHashSet<>(); - columns.add(ExpDataClassTable.Column.Name.name()); - columns.add(ExpDataClassTable.Column.DataCount.name()); - - Map results = new TableSelector(table, columns).getValueMap(String.class); - for (var entry : results.entrySet()) - { - long count = entry.getValue().longValue(); - if (count > 0) - summaries.add(new Summary(count, entry.getKey())); - } - } - - // Sample Types - int sampleTypeCount = SampleTypeService.get().getSampleTypes(c, false).size(); - if (sampleTypeCount > 0) - summaries.add(new Summary(sampleTypeCount, "Sample Type")); - - // Individual Sample Type row counts - { - // The table-level container filter is set to ensure data class types are included - // that may not be defined in the target container but may have rows of data in the target container - TableInfo table = ExpSchema.TableType.SampleSets.createTable(expSchema, null, ContainerFilter.Type.CurrentPlusProjectAndShared.create(c, user)); - - // Issue 51557: The "SampleCount" column is filtered to only count data in the target container - if (table instanceof ExpSampleTypeTableImpl tableImpl) - tableImpl.setSampleCountContainerFilter(ContainerFilter.Type.Current.create(c, user)); - - Set columns = new LinkedHashSet<>(); - columns.add(ExpSampleTypeTable.Column.Name.name()); - columns.add(ExpSampleTypeTable.Column.SampleCount.name()); - - Map results = new TableSelector(table, columns).getValueMap(String.class); - for (var entry : results.entrySet()) - { - long count = entry.getValue().longValue(); - if (count > 0) - { - String name = entry.getKey(); - Summary s = name.equals("MixtureBatches") - ? new Summary(count, "Batch") - : new Summary(count, name); - summaries.add(s); - } - } - } - - return summaries; - } - - @Override - public @NotNull Set> getIntegrationTests() - { - return Set.of( - DomainImpl.TestCase.class, - DomainPropertyImpl.TestCase.class, - ExpDataTableImpl.TestCase.class, - ExperimentServiceImpl.AuditDomainUriTest.class, - ExperimentServiceImpl.LineageQueryTestCase.class, - ExperimentServiceImpl.ParseInputOutputAliasTestCase.class, - ExperimentServiceImpl.TestCase.class, - ExperimentStressTest.class, - LineagePerfTest.class, - LineageTest.class, - OntologyManager.TestCase.class, - PropertyServiceImpl.TestCase.class, - SampleTypeServiceImpl.TestCase.class, - StorageNameGenerator.TestCase.class, - StorageProvisionerImpl.TestCase.class, - UniqueValueCounterTestCase.class, - XarTestPipelineJob.TestCase.class - ); - } - - @Override - public @NotNull Collection>> getIntegrationTestFactories() - { - List>> list = new ArrayList<>(super.getIntegrationTestFactories()); - list.add(new JspTestCase("/org/labkey/experiment/api/ExpDataClassDataTestCase.jsp")); - list.add(new JspTestCase("/org/labkey/experiment/api/ExpSampleTypeTestCase.jsp")); - return list; - } - - @Override - public @NotNull Set> getUnitTests() - { - return Set.of( - GraphAlgorithms.TestCase.class, - LSIDRelativizer.TestCase.class, - Lsid.TestCase.class, - LsidUtils.TestCase.class, - PropertyController.TestCase.class, - Quantity.TestCase.class, - Unit.TestCase.class - ); - } - - @Override - @NotNull - public Collection getSchemaNames() - { - return List.of( - ExpSchema.SCHEMA_NAME, - DataClassDomainKind.PROVISIONED_SCHEMA_NAME, - SampleTypeDomainKind.PROVISIONED_SCHEMA_NAME - ); - } - - @NotNull - @Override - public Collection getProvisionedSchemaNames() - { - return PageFlowUtil.set(DataClassDomainKind.PROVISIONED_SCHEMA_NAME, SampleTypeDomainKind.PROVISIONED_SCHEMA_NAME); - } -} +/* + * Copyright (c) 2008-2019 LabKey Corporation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.labkey.experiment; + +import org.apache.commons.lang3.math.NumberUtils; +import org.jetbrains.annotations.NotNull; +import org.jetbrains.annotations.Nullable; +import org.labkey.api.admin.FolderSerializationRegistry; +import org.labkey.api.assay.AssayProvider; +import org.labkey.api.assay.AssayService; +import org.labkey.api.attachments.AttachmentService; +import org.labkey.api.audit.AuditLogService; +import org.labkey.api.audit.SampleTimelineAuditEvent; +import org.labkey.api.collections.LongHashMap; +import org.labkey.api.data.Container; +import org.labkey.api.data.ContainerFilter; +import org.labkey.api.data.ContainerManager; +import org.labkey.api.data.CoreSchema; +import org.labkey.api.data.DbSchema; +import org.labkey.api.data.DbSchemaType; +import org.labkey.api.data.JdbcType; +import org.labkey.api.data.NameGenerator; +import org.labkey.api.data.SQLFragment; +import org.labkey.api.data.SimpleFilter; +import org.labkey.api.data.SimpleFilter.FilterClause; +import org.labkey.api.data.SqlSelector; +import org.labkey.api.data.TableInfo; +import org.labkey.api.data.TableSelector; +import org.labkey.api.data.UpgradeCode; +import org.labkey.api.defaults.DefaultValueService; +import org.labkey.api.exp.ExperimentException; +import org.labkey.api.exp.ExperimentRunType; +import org.labkey.api.exp.Lsid; +import org.labkey.api.exp.OntologyManager; +import org.labkey.api.exp.PropertyType; +import org.labkey.api.exp.api.DefaultExperimentDataHandler; +import org.labkey.api.exp.api.ExpData; +import org.labkey.api.exp.api.ExpDataClass; +import org.labkey.api.exp.api.ExpLineageService; +import org.labkey.api.exp.api.ExpMaterial; +import org.labkey.api.exp.api.ExpProtocol; +import org.labkey.api.exp.api.ExpProtocolAttachmentType; +import org.labkey.api.exp.api.ExpRunAttachmentType; +import org.labkey.api.exp.api.ExpSampleType; +import org.labkey.api.exp.api.ExperimentJSONConverter; +import org.labkey.api.exp.api.ExperimentService; +import org.labkey.api.exp.api.FilterProtocolInputCriteria; +import org.labkey.api.exp.api.SampleTypeDomainKind; +import org.labkey.api.exp.api.SampleTypeService; +import org.labkey.api.exp.api.StorageProvisioner; +import org.labkey.api.exp.property.DomainAuditProvider; +import org.labkey.api.exp.property.DomainPropertyAuditProvider; +import org.labkey.api.exp.property.ExperimentProperty; +import org.labkey.api.exp.property.PropertyService; +import org.labkey.api.exp.property.SystemProperty; +import org.labkey.api.exp.query.ExpDataClassTable; +import org.labkey.api.exp.query.ExpSampleTypeTable; +import org.labkey.api.exp.query.ExpSchema; +import org.labkey.api.exp.query.SamplesSchema; +import org.labkey.api.exp.xar.LSIDRelativizer; +import org.labkey.api.exp.xar.LsidUtils; +import org.labkey.api.files.FileContentService; +import org.labkey.api.files.TableUpdaterFileListener; +import org.labkey.api.migration.DatabaseMigrationService; +import org.labkey.api.migration.ExperimentDeleteService; +import org.labkey.api.migration.MigrationTableHandler; +import org.labkey.api.module.ModuleContext; +import org.labkey.api.module.ModuleLoader; +import org.labkey.api.module.SpringModule; +import org.labkey.api.module.Summary; +import org.labkey.api.ontology.OntologyService; +import org.labkey.api.ontology.Quantity; +import org.labkey.api.ontology.Unit; +import org.labkey.api.pipeline.PipelineService; +import org.labkey.api.query.FieldKey; +import org.labkey.api.query.FilteredTable; +import org.labkey.api.query.QueryService; +import org.labkey.api.query.UserSchema; +import org.labkey.api.search.SearchService; +import org.labkey.api.security.User; +import org.labkey.api.security.roles.RoleManager; +import org.labkey.api.settings.AppProps; +import org.labkey.api.settings.OptionalFeatureService; +import org.labkey.api.usageMetrics.UsageMetricsService; +import org.labkey.api.util.GUID; +import org.labkey.api.util.JspTestCase; +import org.labkey.api.util.PageFlowUtil; +import org.labkey.api.util.StringUtilsLabKey; +import org.labkey.api.util.SystemMaintenance; +import org.labkey.api.view.AlwaysAvailableWebPartFactory; +import org.labkey.api.view.BaseWebPartFactory; +import org.labkey.api.view.HttpView; +import org.labkey.api.view.JspView; +import org.labkey.api.view.Portal; +import org.labkey.api.view.ViewContext; +import org.labkey.api.view.WebPartFactory; +import org.labkey.api.view.WebPartView; +import org.labkey.api.view.template.WarningService; +import org.labkey.api.vocabulary.security.DesignVocabularyPermission; +import org.labkey.api.webdav.WebdavResource; +import org.labkey.api.webdav.WebdavService; +import org.labkey.experiment.api.DataClassDomainKind; +import org.labkey.experiment.api.ExpDataClassImpl; +import org.labkey.experiment.api.ExpDataClassTableImpl; +import org.labkey.experiment.api.ExpDataClassType; +import org.labkey.experiment.api.ExpDataImpl; +import org.labkey.experiment.api.ExpDataTableImpl; +import org.labkey.experiment.api.ExpMaterialImpl; +import org.labkey.experiment.api.ExpProtocolImpl; +import org.labkey.experiment.api.ExpSampleTypeImpl; +import org.labkey.experiment.api.ExpSampleTypeTableImpl; +import org.labkey.experiment.api.ExperimentServiceImpl; +import org.labkey.experiment.api.ExperimentStressTest; +import org.labkey.experiment.api.GraphAlgorithms; +import org.labkey.experiment.api.LineageTest; +import org.labkey.experiment.api.LogDataType; +import org.labkey.experiment.api.Protocol; +import org.labkey.experiment.api.SampleTypeServiceImpl; +import org.labkey.experiment.api.UniqueValueCounterTestCase; +import org.labkey.experiment.api.VocabularyDomainKind; +import org.labkey.experiment.api.data.ChildOfCompareType; +import org.labkey.experiment.api.data.ChildOfMethod; +import org.labkey.experiment.api.data.LineageCompareType; +import org.labkey.experiment.api.data.ParentOfCompareType; +import org.labkey.experiment.api.data.ParentOfMethod; +import org.labkey.experiment.api.property.DomainImpl; +import org.labkey.experiment.api.property.DomainPropertyImpl; +import org.labkey.experiment.api.property.LengthValidator; +import org.labkey.experiment.api.property.LookupValidator; +import org.labkey.experiment.api.property.PropertyServiceImpl; +import org.labkey.experiment.api.property.RangeValidator; +import org.labkey.experiment.api.property.RegExValidator; +import org.labkey.experiment.api.property.StorageNameGenerator; +import org.labkey.experiment.api.property.StorageProvisionerImpl; +import org.labkey.experiment.api.property.TextChoiceValidator; +import org.labkey.experiment.controllers.exp.ExperimentController; +import org.labkey.experiment.controllers.property.PropertyController; +import org.labkey.experiment.defaults.DefaultValueServiceImpl; +import org.labkey.experiment.lineage.ExpLineageServiceImpl; +import org.labkey.experiment.lineage.LineagePerfTest; +import org.labkey.experiment.pipeline.ExperimentPipelineProvider; +import org.labkey.experiment.pipeline.XarTestPipelineJob; +import org.labkey.experiment.samples.DataClassFolderImporter; +import org.labkey.experiment.samples.DataClassFolderWriter; +import org.labkey.experiment.samples.SampleStatusFolderImporter; +import org.labkey.experiment.samples.SampleTimelineAuditProvider; +import org.labkey.experiment.samples.SampleTypeFolderImporter; +import org.labkey.experiment.samples.SampleTypeFolderWriter; +import org.labkey.experiment.security.DataClassDesignerRole; +import org.labkey.experiment.security.SampleTypeDesignerRole; +import org.labkey.experiment.types.TypesController; +import org.labkey.experiment.xar.FolderXarImporterFactory; +import org.labkey.experiment.xar.FolderXarWriterFactory; + +import java.util.ArrayList; +import java.util.Collection; +import java.util.Collections; +import java.util.HashMap; +import java.util.HashSet; +import java.util.LinkedHashSet; +import java.util.LinkedList; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.function.Supplier; +import java.util.stream.Collectors; + +import static org.labkey.api.data.ColumnRenderPropertiesImpl.STORAGE_UNIQUE_ID_CONCEPT_URI; +import static org.labkey.api.data.ColumnRenderPropertiesImpl.TEXT_CHOICE_CONCEPT_URI; +import static org.labkey.api.exp.api.ExperimentService.MODULE_NAME; +import static org.labkey.api.exp.query.ExpSchema.SAMPLE_FILES_TABLE; + +public class ExperimentModule extends SpringModule +{ + private static final String SAMPLE_TYPE_WEB_PART_NAME = "Sample Types"; + private static final String PROTOCOL_WEB_PART_NAME = "Protocols"; + + public static final String AMOUNT_AND_UNIT_UPGRADE_PROP = "AmountAndUnitAudit"; + public static final String TRANSACTION_ID_PROP = "AuditTransactionId"; + public static final String AUDIT_COUNT_PROP = "AuditRecordCount"; + public static final String EXPERIMENT_RUN_WEB_PART_NAME = "Experiment Runs"; + + @Override + public String getName() + { + return MODULE_NAME; + } + + @Override + public Double getSchemaVersion() + { + return 25.013; + } + + @Nullable + @Override + public UpgradeCode getUpgradeCode() + { + return new ExperimentUpgradeCode(); + } + + @Override + protected void init() + { + addController("experiment", ExperimentController.class); + addController("experiment-types", TypesController.class); + addController("property", PropertyController.class); + ExperimentService.setInstance(new ExperimentServiceImpl()); + SampleTypeService.setInstance(new SampleTypeServiceImpl()); + DefaultValueService.setInstance(new DefaultValueServiceImpl()); + StorageProvisioner.setInstance(StorageProvisionerImpl.get()); + ExpLineageService.setInstance(new ExpLineageServiceImpl()); + + PropertyServiceImpl propertyServiceImpl = new PropertyServiceImpl(); + PropertyService.setInstance(propertyServiceImpl); + UsageMetricsService.get().registerUsageMetrics(getName(), propertyServiceImpl); + + UsageMetricsService.get().registerUsageMetrics(getName(), FileLinkMetricsProvider.getInstance()); + + ExperimentProperty.register(); + SamplesSchema.register(this); + ExpSchema.register(this); + + PropertyService.get().registerDomainKind(new SampleTypeDomainKind()); + PropertyService.get().registerDomainKind(new DataClassDomainKind()); + PropertyService.get().registerDomainKind(new VocabularyDomainKind()); + + QueryService.get().addCompareType(new ChildOfCompareType()); + QueryService.get().addCompareType(new ParentOfCompareType()); + QueryService.get().addCompareType(new LineageCompareType()); + QueryService.get().registerMethod(ChildOfMethod.NAME, new ChildOfMethod(), JdbcType.BOOLEAN, 2, 3); + QueryService.get().registerMethod(ParentOfMethod.NAME, new ParentOfMethod(), JdbcType.BOOLEAN, 2, 3); + QueryService.get().addQueryListener(new ExperimentQueryChangeListener()); + QueryService.get().addQueryListener(new PropertyQueryChangeListener()); + + PropertyService.get().registerValidatorKind(new RegExValidator()); + PropertyService.get().registerValidatorKind(new RangeValidator()); + PropertyService.get().registerValidatorKind(new LookupValidator()); + PropertyService.get().registerValidatorKind(new LengthValidator()); + PropertyService.get().registerValidatorKind(new TextChoiceValidator()); + + ExperimentService.get().registerExperimentDataHandler(new DefaultExperimentDataHandler()); + ExperimentService.get().registerProtocolInputCriteria(new FilterProtocolInputCriteria.Factory()); + ExperimentService.get().registerNameExpressionType("sampletype", "exp", "MaterialSource", "nameexpression"); + ExperimentService.get().registerNameExpressionType("aliquots", "exp", "MaterialSource", "aliquotnameexpression"); + ExperimentService.get().registerNameExpressionType("dataclass", "exp", "DataClass", "nameexpression"); + + OptionalFeatureService.get().addExperimentalFeatureFlag(AppProps.EXPERIMENTAL_RESOLVE_PROPERTY_URI_COLUMNS, "Resolve property URIs as columns on experiment tables", + "If a column is not found on an experiment table, attempt to resolve the column name as a Property URI and add it as a property column", false); + if (CoreSchema.getInstance().getSqlDialect().isSqlServer()) + { + OptionalFeatureService.get().addExperimentalFeatureFlag(NameGenerator.EXPERIMENTAL_WITH_COUNTER, "Use strict incremental withCounter and rootSampleCount expression", + "When withCounter or rootSampleCount is used in name expression, make sure the count increments one-by-one and does not jump.", true); + } + else + { + OptionalFeatureService.get().addExperimentalFeatureFlag(SAMPLE_FILES_TABLE, "Manage Stale Sample Files", + "Allow 'Manage Sample Files' to view sample files that are no longer referenced by samples", false); + + OptionalFeatureService.get().addExperimentalFeatureFlag(NameGenerator.EXPERIMENTAL_ALLOW_GAP_COUNTER, "Allow gap with withCounter and rootSampleCount expression", + "Check this option if gaps in the count generated by withCounter or rootSampleCount name expression are allowed.", true); + } + OptionalFeatureService.get().addExperimentalFeatureFlag(AppProps.QUANTITY_COLUMN_SUFFIX_TESTING, "Quantity column suffix testing", + "If a column name contains a \"__\" suffix, this feature allows for testing it as a Quantity display column", false); + + RoleManager.registerPermission(new DesignVocabularyPermission(), true); + RoleManager.registerRole(new SampleTypeDesignerRole()); + RoleManager.registerRole(new DataClassDesignerRole()); + + AttachmentService.get().registerAttachmentType(ExpRunAttachmentType.get()); + AttachmentService.get().registerAttachmentType(ExpProtocolAttachmentType.get()); + + WebdavService.get().addExpDataProvider((path, container) -> ExperimentService.get().getAllExpDataByURL(path, container)); + ExperimentService.get().registerObjectReferencer(ExperimentServiceImpl.get()); + + addModuleProperty(new LineageMaximumDepthModuleProperty(this)); + WarningService.get().register(new ExperimentWarningProvider()); + } + + @Override + public boolean hasScripts() + { + return true; + } + + @Override + @NotNull + protected Collection createWebPartFactories() + { + List result = new ArrayList<>(); + + BaseWebPartFactory runGroupsFactory = new BaseWebPartFactory(RunGroupWebPart.WEB_PART_NAME, WebPartFactory.LOCATION_BODY, WebPartFactory.LOCATION_RIGHT) + { + @Override + public WebPartView getWebPartView(@NotNull ViewContext portalCtx, @NotNull Portal.WebPart webPart) + { + return new RunGroupWebPart(portalCtx, WebPartFactory.LOCATION_RIGHT.equalsIgnoreCase(webPart.getLocation()), webPart); + } + }; + runGroupsFactory.addLegacyNames("Experiments", "Experiment", "Experiment Navigator", "Narrow Experiments"); + result.add(runGroupsFactory); + + BaseWebPartFactory runTypesFactory = new BaseWebPartFactory(RunTypeWebPart.WEB_PART_NAME, WebPartFactory.LOCATION_BODY, WebPartFactory.LOCATION_RIGHT) + { + @Override + public WebPartView getWebPartView(@NotNull ViewContext portalCtx, @NotNull Portal.WebPart webPart) + { + return new RunTypeWebPart(); + } + }; + result.add(runTypesFactory); + + result.add(new ExperimentRunWebPartFactory()); + BaseWebPartFactory sampleTypeFactory = new BaseWebPartFactory(SAMPLE_TYPE_WEB_PART_NAME, WebPartFactory.LOCATION_BODY, WebPartFactory.LOCATION_RIGHT) + { + @Override + public WebPartView getWebPartView(@NotNull ViewContext portalCtx, @NotNull Portal.WebPart webPart) + { + return new SampleTypeWebPart(WebPartFactory.LOCATION_RIGHT.equalsIgnoreCase(webPart.getLocation()), portalCtx); + } + }; + sampleTypeFactory.addLegacyNames("Narrow Sample Sets", "Sample Sets"); + result.add(sampleTypeFactory); + result.add(new AlwaysAvailableWebPartFactory("Samples Menu", false, false, WebPartFactory.LOCATION_MENUBAR) { + @Override + public WebPartView getWebPartView(@NotNull ViewContext portalCtx, @NotNull Portal.WebPart webPart) + { + WebPartView view = new JspView<>("/org/labkey/experiment/samplesAndAnalytes.jsp", webPart); + view.setTitle("Samples"); + return view; + } + }); + + result.add(new AlwaysAvailableWebPartFactory("Data Classes", false, false, WebPartFactory.LOCATION_BODY, WebPartFactory.LOCATION_RIGHT) { + @Override + public WebPartView getWebPartView(@NotNull ViewContext portalCtx, @NotNull Portal.WebPart webPart) + { + return new DataClassWebPart(WebPartFactory.LOCATION_RIGHT.equalsIgnoreCase(webPart.getLocation()), portalCtx, webPart); + } + }); + + BaseWebPartFactory narrowProtocolFactory = new BaseWebPartFactory(PROTOCOL_WEB_PART_NAME, WebPartFactory.LOCATION_RIGHT) + { + @Override + public WebPartView getWebPartView(@NotNull ViewContext portalCtx, @NotNull Portal.WebPart webPart) + { + return new ProtocolWebPart(WebPartFactory.LOCATION_RIGHT.equalsIgnoreCase(webPart.getLocation()), portalCtx); + } + }; + narrowProtocolFactory.addLegacyNames("Narrow Protocols"); + result.add(narrowProtocolFactory); + + return result; + } + + private void addDataResourceResolver(String categoryName) + { + SearchService.get().addResourceResolver(categoryName, new SearchService.ResourceResolver() + { + @Override + public WebdavResource resolve(@NotNull String resourceIdentifier) + { + ExpDataImpl data = ExpDataImpl.fromDocumentId(resourceIdentifier); + if (data == null) + return null; + + return data.createIndexDocument(null); + } + + @Override + public Map getCustomSearchJson(User user, @NotNull String resourceIdentifier) + { + ExpDataImpl data = ExpDataImpl.fromDocumentId(resourceIdentifier); + if (data == null) + return null; + + return ExperimentJSONConverter.serializeData(data, user, ExperimentJSONConverter.DEFAULT_SETTINGS).toMap(); + } + + @Override + public Map> getCustomSearchJsonMap(User user, @NotNull Collection resourceIdentifiers) + { + Map idDataMap = ExpDataImpl.fromDocumentIds(resourceIdentifiers); + if (idDataMap == null) + return null; + + Map> searchJsonMap = new HashMap<>(); + for (String resourceIdentifier : idDataMap.keySet()) + searchJsonMap.put(resourceIdentifier, ExperimentJSONConverter.serializeData(idDataMap.get(resourceIdentifier), user, ExperimentJSONConverter.DEFAULT_SETTINGS).toMap()); + return searchJsonMap; + } + }); + } + + private void addDataClassResourceResolver(String categoryName) + { + SearchService.get().addResourceResolver(categoryName, new SearchService.ResourceResolver(){ + @Override + public Map getCustomSearchJson(User user, @NotNull String resourceIdentifier) + { + int rowId = NumberUtils.toInt(resourceIdentifier.replace(categoryName + ":", "")); + if (rowId == 0) + return null; + + ExpDataClass dataClass = ExperimentService.get().getDataClass(rowId); + if (dataClass == null) + return null; + + Map properties = ExperimentJSONConverter.serializeExpObject(dataClass, null, ExperimentJSONConverter.DEFAULT_SETTINGS, user).toMap(); + + //Need to map to proper Icon + properties.put("type", "dataClass" + (dataClass.getCategory() != null ? ":" + dataClass.getCategory() : "")); + + return properties; + } + }); + } + + private void addSampleTypeResourceResolver(String categoryName) + { + SearchService.get().addResourceResolver(categoryName, new SearchService.ResourceResolver(){ + @Override + public Map getCustomSearchJson(User user, @NotNull String resourceIdentifier) + { + int rowId = NumberUtils.toInt(resourceIdentifier.replace(categoryName + ":", "")); + if (rowId == 0) + return null; + + ExpSampleType sampleType = SampleTypeService.get().getSampleType(rowId); + if (sampleType == null) + return null; + + Map properties = ExperimentJSONConverter.serializeExpObject(sampleType, null, ExperimentJSONConverter.DEFAULT_SETTINGS, user).toMap(); + + //Need to map to proper Icon + properties.put("type", "sampleSet"); + + return properties; + } + }); + } + + private void addSampleResourceResolver(String categoryName) + { + SearchService.get().addResourceResolver(categoryName, new SearchService.ResourceResolver(){ + @Override + public Map getCustomSearchJson(User user, @NotNull String resourceIdentifier) + { + int rowId = NumberUtils.toInt(resourceIdentifier.replace(categoryName + ":", "")); + if (rowId == 0) + return null; + + ExpMaterial material = ExperimentService.get().getExpMaterial(rowId); + if (material == null) + return null; + + return ExperimentJSONConverter.serializeMaterial(material, user, ExperimentJSONConverter.DEFAULT_SETTINGS).toMap(); + } + + @Override + public Map> getCustomSearchJsonMap(User user, @NotNull Collection resourceIdentifiers) + { + Set rowIds = new HashSet<>(); + Map rowIdIdentifierMap = new LongHashMap<>(); + for (String resourceIdentifier : resourceIdentifiers) + { + long rowId = NumberUtils.toLong(resourceIdentifier.replace(categoryName + ":", "")); + if (rowId != 0) + { + rowIds.add(rowId); + rowIdIdentifierMap.put(rowId, resourceIdentifier); + } + } + + Map> searchJsonMap = new HashMap<>(); + for (ExpMaterial material : ExperimentService.get().getExpMaterials(rowIds)) + { + searchJsonMap.put( + rowIdIdentifierMap.get(material.getRowId()), + ExperimentJSONConverter.serializeMaterial(material, user, ExperimentJSONConverter.DEFAULT_SETTINGS).toMap() + ); + } + + return searchJsonMap; + } + }); + } + + @Override + protected void startupAfterSpringConfig(ModuleContext moduleContext) + { + SearchService ss = SearchService.get(); +// ss.addSearchCategory(OntologyManager.conceptCategory); + ss.addSearchCategory(ExpSampleTypeImpl.searchCategory); + ss.addSearchCategory(ExpSampleTypeImpl.mediaSearchCategory); + ss.addSearchCategory(ExpMaterialImpl.searchCategory); + ss.addSearchCategory(ExpMaterialImpl.mediaSearchCategory); + ss.addSearchCategory(ExpDataClassImpl.SEARCH_CATEGORY); + ss.addSearchCategory(ExpDataClassImpl.MEDIA_SEARCH_CATEGORY); + ss.addSearchCategory(ExpDataImpl.expDataCategory); + ss.addSearchCategory(ExpDataImpl.expMediaDataCategory); + ss.addSearchResultTemplate(new ExpDataImpl.DataSearchResultTemplate()); + addDataResourceResolver(ExpDataImpl.expDataCategory.getName()); + addDataResourceResolver(ExpDataImpl.expMediaDataCategory.getName()); + addDataClassResourceResolver(ExpDataClassImpl.SEARCH_CATEGORY.getName()); + addDataClassResourceResolver(ExpDataClassImpl.MEDIA_SEARCH_CATEGORY.getName()); + addSampleTypeResourceResolver(ExpSampleTypeImpl.searchCategory.getName()); + addSampleTypeResourceResolver(ExpSampleTypeImpl.mediaSearchCategory.getName()); + addSampleResourceResolver(ExpMaterialImpl.searchCategory.getName()); + addSampleResourceResolver(ExpMaterialImpl.mediaSearchCategory.getName()); + ss.addDocumentProvider(ExperimentServiceImpl.get()); + + PipelineService.get().registerPipelineProvider(new ExperimentPipelineProvider(this)); + ExperimentService.get().registerExperimentRunTypeSource(container -> Collections.singleton(ExperimentRunType.ALL_RUNS_TYPE)); + ExperimentService.get().registerDataType(new LogDataType()); + + AuditLogService.get().registerAuditType(new DomainAuditProvider()); + AuditLogService.get().registerAuditType(new DomainPropertyAuditProvider()); + AuditLogService.get().registerAuditType(new ExperimentAuditProvider()); + AuditLogService.get().registerAuditType(new SampleTypeAuditProvider()); + AuditLogService.get().registerAuditType(new SampleTimelineAuditProvider()); + + FileContentService fileContentService = FileContentService.get(); + if (null != fileContentService) + { + fileContentService.addFileListener(new ExpDataFileListener()); + fileContentService.addFileListener(new TableUpdaterFileListener(ExperimentService.get().getTinfoExperimentRun(), "FilePathRoot", TableUpdaterFileListener.Type.fileRootPath, "RowId")); + fileContentService.addFileListener(new FileLinkFileListener()); + } + ContainerManager.addContainerListener(new ContainerManager.ContainerListener() + { + @Override + public void containerDeleted(Container c, User user) + { + try + { + ExperimentService.get().deleteAllExpObjInContainer(c, user); + } + catch (ExperimentException ee) + { + throw new RuntimeException(ee); + } + } + }, + // This is in the Last group because when a container is deleted, + // the Experiment listener needs to be called after the Study listener, + // because Study needs the metadata held by Experiment to delete properly. + // but it should be before the CoreContainerListener + ContainerManager.ContainerListener.Order.Last); + + if (ModuleLoader.getInstance().shouldInsertData()) + SystemProperty.registerProperties(); + + FolderSerializationRegistry folderRegistry = FolderSerializationRegistry.get(); + if (null != folderRegistry) + { + folderRegistry.addFactories(new FolderXarWriterFactory(), new FolderXarImporterFactory()); + folderRegistry.addWriterFactory(new SampleTypeFolderWriter.SampleTypeDesignWriter.Factory()); + folderRegistry.addWriterFactory(new SampleTypeFolderWriter.SampleTypeDataWriter.Factory()); + folderRegistry.addWriterFactory(new DataClassFolderWriter.DataClassDesignWriter.Factory()); + folderRegistry.addWriterFactory(new DataClassFolderWriter.DataClassDataWriter.Factory()); + folderRegistry.addImportFactory(new SampleTypeFolderImporter.Factory()); + folderRegistry.addImportFactory(new DataClassFolderImporter.Factory()); + folderRegistry.addImportFactory(new SampleStatusFolderImporter.Factory()); + } + + AttachmentService.get().registerAttachmentType(ExpDataClassType.get()); + + WebdavService.get().addProvider(new ScriptsResourceProvider()); + + SystemMaintenance.addTask(new FileLinkMetricsMaintenanceTask()); + + UsageMetricsService svc = UsageMetricsService.get(); + if (null != svc) + { + svc.registerUsageMetrics(getName(), () -> { + Map results = new HashMap<>(); + + DbSchema schema = ExperimentService.get().getSchema(); + if (AssayService.get() != null) + { + Map assayMetrics = new HashMap<>(); + SQLFragment baseRunSQL = new SQLFragment("SELECT COUNT(*) FROM ").append(ExperimentService.get().getTinfoExperimentRun(), "r").append(" WHERE lsid LIKE ?"); + SQLFragment baseProtocolSQL = new SQLFragment("SELECT * FROM ").append(ExperimentService.get().getTinfoProtocol(), "p").append(" WHERE lsid LIKE ? AND ApplicationType = ?"); + for (AssayProvider assayProvider : AssayService.get().getAssayProviders()) + { + Map protocolMetrics = new HashMap<>(); + + // Run count across all assay designs of this type + SQLFragment runSQL = new SQLFragment(baseRunSQL); + runSQL.add(Lsid.namespaceLikeString(assayProvider.getRunLSIDPrefix())); + protocolMetrics.put("runCount", new SqlSelector(schema, runSQL).getObject(Long.class)); + + // Number of assay designs of this type + SQLFragment protocolSQL = new SQLFragment(baseProtocolSQL); + protocolSQL.add(assayProvider.getProtocolPattern()); + protocolSQL.add(ExpProtocol.ApplicationType.ExperimentRun.toString()); + List protocols = new SqlSelector(schema, protocolSQL).getArrayList(Protocol.class); + protocolMetrics.put("protocolCount", protocols.size()); + + List wrappedProtocols = protocols.stream().map(ExpProtocolImpl::new).collect(Collectors.toList()); + + protocolMetrics.put("resultRowCount", assayProvider.getResultRowCount(wrappedProtocols)); + + // Primary implementation class + protocolMetrics.put("implementingClass", assayProvider.getClass()); + + assayMetrics.put(assayProvider.getName(), protocolMetrics); + } + assayMetrics.put("autoLinkedAssayCount", new SqlSelector(schema, "SELECT COUNT(*) FROM exp.protocol EP JOIN exp.objectPropertiesView OP ON EP.lsid = OP.objecturi WHERE OP.propertyuri = 'terms.labkey.org#AutoCopyTargetContainer'").getObject(Long.class)); + assayMetrics.put("protocolsWithTransformScriptCount", new SqlSelector(schema, "SELECT COUNT(*) FROM exp.protocol EP JOIN exp.objectPropertiesView OP ON EP.lsid = OP.objecturi WHERE OP.name = 'TransformScript' AND status = 'Active'").getObject(Long.class)); + assayMetrics.put("protocolsWithTransformScriptRunOnEditCount", new SqlSelector(schema, "SELECT COUNT(*) FROM exp.protocol EP JOIN exp.objectPropertiesView OP ON EP.lsid = OP.objecturi WHERE OP.name = 'TransformScript' AND status = 'Active' AND OP.stringvalue LIKE '%\"INSERT\"%'").getObject(Long.class)); + assayMetrics.put("protocolsWithTransformScriptRunOnImportCount", new SqlSelector(schema, "SELECT COUNT(*) FROM exp.protocol EP JOIN exp.objectPropertiesView OP ON EP.lsid = OP.objecturi WHERE OP.name = 'TransformScript' AND status = 'Active' AND OP.stringvalue LIKE '%\"INSERT\"%'").getObject(Long.class)); + + assayMetrics.put("standardAssayWithPlateSupportCount", new SqlSelector(schema, "SELECT COUNT(*) FROM exp.protocol EP JOIN exp.objectPropertiesView OP ON EP.lsid = OP.objecturi WHERE OP.name = 'PlateMetadata' AND floatValue = 1").getObject(Long.class)); + SQLFragment runsWithPlateSQL = new SQLFragment(""" + SELECT COUNT(*) FROM exp.experimentrun r + INNER JOIN exp.object o ON o.objectUri = r.lsid + INNER JOIN exp.objectproperty op ON op.objectId = o.objectId + WHERE op.propertyid IN ( + SELECT propertyid FROM exp.propertydescriptor WHERE name = ? AND lookupquery = ? + )"""); + assayMetrics.put("standardAssayRunsWithPlateTemplate", new SqlSelector(schema, new SQLFragment(runsWithPlateSQL).add("PlateTemplate").add("PlateTemplate")).getObject(Long.class)); + assayMetrics.put("standardAssayRunsWithPlateSet", new SqlSelector(schema, new SQLFragment(runsWithPlateSQL).add("PlateSet").add("PlateSet")).getObject(Long.class)); + + assayMetrics.put("assayRunsFileColumnCount", new SqlSelector(schema, """ + SELECT COUNT(DISTINCT DD.DomainURI) FROM + exp.PropertyDescriptor D\s + JOIN exp.PropertyDomain PD ON D.propertyId = PD.propertyid + JOIN exp.DomainDescriptor DD on PD.domainID = DD.domainId + WHERE DD.domainUri LIKE ? AND D.rangeURI = ?""", "urn:lsid:%:" + ExpProtocol.AssayDomainTypes.Run.getPrefix() + ".%", PropertyType.FILE_LINK.getTypeUri()).getObject(Long.class)); + + assayMetrics.put("assayResultsFileColumnCount", new SqlSelector(schema, """ + SELECT COUNT(DISTINCT DD.DomainURI) FROM + exp.PropertyDescriptor D\s + JOIN exp.PropertyDomain PD ON D.propertyId = PD.propertyid + JOIN exp.DomainDescriptor DD on PD.domainID = DD.domainId + WHERE DD.domainUri LIKE ? AND D.rangeURI = ?""", "urn:lsid:%:" + ExpProtocol.AssayDomainTypes.Result.getPrefix() + ".%", PropertyType.FILE_LINK.getTypeUri()).getObject(Long.class)); + + Map sampleLookupCountMetrics = new HashMap<>(); + SQLFragment baseAssaySampleLookupSQL = new SQLFragment("SELECT COUNT(*) FROM exp.propertydescriptor WHERE (lookupschema = 'samples' OR (lookupschema = 'exp' AND lookupquery = 'Materials')) AND propertyuri LIKE ?"); + + SQLFragment batchAssaySampleLookupSQL = new SQLFragment(baseAssaySampleLookupSQL); + batchAssaySampleLookupSQL.add("urn:lsid:%:" + ExpProtocol.AssayDomainTypes.Batch.getPrefix() + ".%"); + sampleLookupCountMetrics.put("batchDomain", new SqlSelector(schema, batchAssaySampleLookupSQL).getObject(Long.class)); + + SQLFragment runAssaySampleLookupSQL = new SQLFragment(baseAssaySampleLookupSQL); + runAssaySampleLookupSQL.add("urn:lsid:%:" + ExpProtocol.AssayDomainTypes.Run.getPrefix() + ".%"); + sampleLookupCountMetrics.put("runDomain", new SqlSelector(schema, runAssaySampleLookupSQL).getObject(Long.class)); + + SQLFragment resultAssaySampleLookupSQL = new SQLFragment(baseAssaySampleLookupSQL); + resultAssaySampleLookupSQL.add("urn:lsid:%:" + ExpProtocol.AssayDomainTypes.Result.getPrefix() + ".%"); + sampleLookupCountMetrics.put("resultDomain", new SqlSelector(schema, resultAssaySampleLookupSQL).getObject(Long.class)); + + SQLFragment resultAssayMultipleSampleLookupSQL = new SQLFragment( + """ + SELECT COUNT(*) FROM ( + SELECT PD.domainid, COUNT(*) AS PropCount + FROM exp.propertydescriptor D + JOIN exp.PropertyDomain PD ON D.propertyId = PD.propertyid + WHERE (lookupschema = 'samples' OR (lookupschema = 'exp' AND lookupquery = 'Materials')) + AND propertyuri LIKE ? + GROUP BY PD.domainid + ) X WHERE X.PropCount > 1""" + ); + resultAssayMultipleSampleLookupSQL.add("urn:lsid:%:" + ExpProtocol.AssayDomainTypes.Result.getPrefix() + ".%"); + sampleLookupCountMetrics.put("resultDomainWithMultiple", new SqlSelector(schema, resultAssayMultipleSampleLookupSQL).getObject(Long.class)); + + assayMetrics.put("sampleLookupCount", sampleLookupCountMetrics); + + + // Putting these metrics at the same level as the other BooleanColumnCount metrics (e.g., sampleTypeWithBooleanColumnCount) + results.put("assayResultWithBooleanColumnCount", new SqlSelector(schema, """ + SELECT COUNT(DISTINCT DD.DomainURI) FROM + exp.PropertyDescriptor D\s + JOIN exp.PropertyDomain PD ON D.propertyId = PD.propertyid + JOIN exp.DomainDescriptor DD on PD.domainID = DD.domainId + WHERE D.propertyURI LIKE ? AND D.rangeURI = ?""", "urn:lsid:%:" + ExpProtocol.AssayDomainTypes.Result.getPrefix() + ".%", PropertyType.BOOLEAN.getTypeUri()).getObject(Long.class)); + + results.put("assayRunWithBooleanColumnCount", new SqlSelector(schema, """ + SELECT COUNT(DISTINCT DD.DomainURI) FROM + exp.PropertyDescriptor D\s + JOIN exp.PropertyDomain PD ON D.propertyId = PD.propertyid + JOIN exp.DomainDescriptor DD on PD.domainID = DD.domainId + WHERE D.propertyURI LIKE ? AND D.rangeURI = ?""", "urn:lsid:%:" + ExpProtocol.AssayDomainTypes.Run.getPrefix() + ".%", PropertyType.BOOLEAN.getTypeUri()).getObject(Long.class)); + + results.put("assay", assayMetrics); + } + + results.put("autoLinkedSampleSetCount", new SqlSelector(schema, "SELECT COUNT(*) FROM exp.materialsource WHERE autoLinkTargetContainer IS NOT NULL").getObject(Long.class)); + results.put("sampleSetCount", new SqlSelector(schema, "SELECT COUNT(*) FROM exp.materialsource").getObject(Long.class)); + + if (schema.getSqlDialect().isPostgreSQL()) // SQLServer does not support regular expression queries + { + Collection> numSampleCounts = new SqlSelector(schema, """ + SELECT totalCount, numberNameCount FROM + (SELECT cpastype, COUNT(*) AS totalCount from exp.material GROUP BY cpastype) t + JOIN + (SELECT cpastype, COUNT(*) AS numberNameCount FROM exp.material m WHERE m.name SIMILAR TO '[0-9.]*' GROUP BY cpastype) ns + ON t.cpastype = ns.cpastype""").getMapCollection(); + results.put("sampleSetWithNumberNamesCount", numSampleCounts.size()); + results.put("sampleSetWithOnlyNumberNamesCount", numSampleCounts.stream().filter( + map -> (Long) map.get("totalCount") > 0 && map.get("totalCount") == map.get("numberNameCount") + ).count()); + } + UserSchema userSchema = AuditLogService.getAuditLogSchema(User.getSearchUser(), ContainerManager.getRoot()); + FilteredTable table = (FilteredTable) userSchema.getTable(SampleTimelineAuditEvent.EVENT_TYPE); + + SQLFragment sql = new SQLFragment("SELECT COUNT(*)\n" + + " FROM (\n" + + " -- updates that are marked as lineage updates\n" + + " (SELECT DISTINCT transactionId\n" + + " FROM " + table.getRealTable().getFromSQL("").getSQL() +"\n" + + " WHERE islineageupdate = " + schema.getSqlDialect().getBooleanTRUE() + "\n" + + " AND comment = 'Sample was updated.'\n" + + " ) a1\n" + + " JOIN\n" + + " -- but have associated entries that are not lineage updates\n" + + " (SELECT DISTINCT transactionid\n" + + " FROM " + table.getRealTable().getFromSQL("").getSQL() + "\n" + + " WHERE islineageupdate = " + schema.getSqlDialect().getBooleanFALSE() + ") a2\n" + + " ON a1.transactionid = a2.transactionid\n" + + " )"); + + results.put("sampleLineageAuditDiscrepancyCount", new SqlSelector(schema, sql.getSQL()).getObject(Long.class)); + + results.put("sampleCount", new SqlSelector(schema, "SELECT COUNT(*) FROM exp.material").getObject(Long.class)); + results.put("aliquotCount", new SqlSelector(schema, "SELECT COUNT(*) FROM exp.material where aliquotedfromlsid IS NOT NULL").getObject(Long.class)); + results.put("sampleNullAmountCount", new SqlSelector(schema, "SELECT COUNT(*) FROM exp.material WHERE storedamount IS NULL").getObject(Long.class)); + results.put("sampleNegativeAmountCount", new SqlSelector(schema, "SELECT COUNT(*) FROM exp.material WHERE storedamount < 0").getObject(Long.class)); + results.put("sampleUnitsDifferCount", new SqlSelector(schema, "SELECT COUNT(*) from exp.material m JOIN exp.materialSource s ON m.materialsourceid = s.rowid WHERE m.units != s.metricunit").getObject(Long.class)); + results.put("sampleTypesWithoutUnitsCount", new SqlSelector(schema, "SELECT COUNT(*) from exp.materialSource WHERE category IS NULL AND metricunit IS NULL").getObject(Long.class)); + + results.put("duplicateSampleMaterialNameCount", new SqlSelector(schema, "SELECT COUNT(*) as duplicateCount FROM " + + "(SELECT name, cpastype FROM exp.material WHERE cpastype <> 'Material' GROUP BY name, cpastype HAVING COUNT(*) > 1) d").getObject(Long.class)); + results.put("duplicateSpecimenMaterialNameCount", new SqlSelector(schema, "SELECT COUNT(*) as duplicateCount FROM " + + "(SELECT name, cpastype FROM exp.material WHERE cpastype = 'Material' GROUP BY name, cpastype HAVING COUNT(*) > 1) d").getObject(Long.class)); + String duplicateCaseInsensitiveSampleNameCountSql = """ + SELECT COUNT(*) FROM + ( + SELECT 1 AS found + FROM exp.material + WHERE materialsourceid IS NOT NULL + GROUP BY LOWER(name), materialsourceid + HAVING COUNT(*) > 1 + ) AS duplicates + """; + String duplicateCaseInsensitiveDataNameCountSql = """ + SELECT COUNT(*) FROM + ( + SELECT 1 AS found + FROM exp.data + WHERE classid IS NOT NULL + GROUP BY LOWER(name), classid + HAVING COUNT(*) > 1 + ) AS duplicates + """; + results.put("duplicateCaseInsensitiveSampleNameCount", new SqlSelector(schema, duplicateCaseInsensitiveSampleNameCountSql).getObject(Long.class)); + results.put("duplicateCaseInsensitiveDataNameCount", new SqlSelector(schema, duplicateCaseInsensitiveDataNameCountSql).getObject(Long.class)); + + results.put("dataClassCount", new SqlSelector(schema, "SELECT COUNT(*) FROM exp.dataclass").getObject(Long.class)); + results.put("dataClassRowCount", new SqlSelector(schema, "SELECT COUNT(*) FROM exp.data WHERE classid IN (SELECT rowid FROM exp.dataclass)").getObject(Long.class)); + results.put("dataWithDataParentsCount", new SqlSelector(schema, "SELECT COUNT(DISTINCT d.sourceApplicationId) FROM exp.data d\n" + + "JOIN exp.datainput di ON di.targetapplicationid = d.sourceapplicationid").getObject(Long.class)); + if (schema.getSqlDialect().isPostgreSQL()) + { + Collection> numDataClassObjectsCounts = new SqlSelector(schema, """ + SELECT totalCount, numberNameCount FROM + (SELECT cpastype, COUNT(*) AS totalCount from exp.data GROUP BY cpastype) t + JOIN + (SELECT cpastype, COUNT(*) AS numberNameCount FROM exp.data m WHERE m.name SIMILAR TO '[0-9.]*' GROUP BY cpastype) ns + ON t.cpastype = ns.cpastype""").getMapCollection(); + results.put("dataClassWithNumberNamesCount", numDataClassObjectsCounts.size()); + results.put("dataClassWithOnlyNumberNamesCount", numDataClassObjectsCounts.stream().filter(map -> + (Long) map.get("totalCount") > 0 && map.get("totalCount") == map.get("numberNameCount")).count()); + } + + results.put("ontologyPrincipalConceptCodeCount", new SqlSelector(schema, "SELECT COUNT(*) FROM exp.propertydescriptor WHERE principalconceptcode IS NOT NULL").getObject(Long.class)); + results.put("ontologyLookupColumnCount", new SqlSelector(schema, "SELECT COUNT(*) FROM exp.propertydescriptor WHERE concepturi = ?", OntologyService.conceptCodeConceptURI).getObject(Long.class)); + results.put("ontologyConceptSubtreeCount", new SqlSelector(schema, "SELECT COUNT(*) FROM exp.propertydescriptor WHERE conceptsubtree IS NOT NULL").getObject(Long.class)); + results.put("ontologyConceptImportColumnCount", new SqlSelector(schema, "SELECT COUNT(*) FROM exp.propertydescriptor WHERE conceptimportcolumn IS NOT NULL").getObject(Long.class)); + results.put("ontologyConceptLabelColumnCount", new SqlSelector(schema, "SELECT COUNT(*) FROM exp.propertydescriptor WHERE conceptlabelcolumn IS NOT NULL").getObject(Long.class)); + + results.put("scannableColumnCount", new SqlSelector(schema, "SELECT COUNT(*) FROM exp.propertydescriptor WHERE scannable = ?", true).getObject(Long.class)); + results.put("uniqueIdColumnCount", new SqlSelector(schema, "SELECT COUNT(*) FROM exp.propertydescriptor WHERE concepturi = ?", STORAGE_UNIQUE_ID_CONCEPT_URI).getObject(Long.class)); + results.put("sampleTypeWithUniqueIdCount", new SqlSelector(schema, """ + SELECT COUNT(DISTINCT DD.DomainURI) FROM + exp.PropertyDescriptor D\s + JOIN exp.PropertyDomain PD ON D.propertyId = PD.propertyid + JOIN exp.DomainDescriptor DD on PD.domainID = DD.domainId + WHERE D.conceptURI = ?""", STORAGE_UNIQUE_ID_CONCEPT_URI).getObject(Long.class)); + + results.put("fileColumnCount", new SqlSelector(schema, "SELECT COUNT(*) FROM exp.propertydescriptor WHERE rangeURI = ?", PropertyType.FILE_LINK.getTypeUri()).getObject(Long.class)); + results.put("sampleTypeWithFileColumnCount", new SqlSelector(schema, """ + SELECT COUNT(DISTINCT DD.DomainURI) FROM + exp.PropertyDescriptor D\s + JOIN exp.PropertyDomain PD ON D.propertyId = PD.propertyid + JOIN exp.DomainDescriptor DD on PD.domainID = DD.domainId + WHERE DD.storageSchemaName = ? AND D.rangeURI = ?""", SampleTypeDomainKind.PROVISIONED_SCHEMA_NAME, PropertyType.FILE_LINK.getTypeUri()).getObject(Long.class)); + results.put("sampleTypeWithBooleanColumnCount", new SqlSelector(schema, """ + SELECT COUNT(DISTINCT DD.DomainURI) FROM + exp.PropertyDescriptor D\s + JOIN exp.PropertyDomain PD ON D.propertyId = PD.propertyid + JOIN exp.DomainDescriptor DD on PD.domainID = DD.domainId + WHERE DD.storageSchemaName = ? AND D.rangeURI = ?""", SampleTypeDomainKind.PROVISIONED_SCHEMA_NAME, PropertyType.BOOLEAN.getTypeUri()).getObject(Long.class)); + + results.put("sampleTypeAliquotSpecificField", new SqlSelector(schema, """ + SELECT COUNT(DISTINCT D.PropertyURI) FROM + exp.PropertyDescriptor D\s + JOIN exp.PropertyDomain PD ON D.propertyId = PD.propertyid + JOIN exp.DomainDescriptor DD on PD.domainID = DD.domainId + WHERE DD.storageSchemaName = ? AND D.derivationDataScope = ?""", SampleTypeDomainKind.PROVISIONED_SCHEMA_NAME, ExpSchema.DerivationDataScopeType.ChildOnly.name()).getObject(Long.class)); + results.put("sampleTypeParentOnlyField", new SqlSelector(schema, """ + SELECT COUNT(DISTINCT D.PropertyURI) FROM + exp.PropertyDescriptor D\s + JOIN exp.PropertyDomain PD ON D.propertyId = PD.propertyid + JOIN exp.DomainDescriptor DD on PD.domainID = DD.domainId + WHERE DD.storageSchemaName = ? AND (D.derivationDataScope = ? OR D.derivationDataScope IS NULL)""", SampleTypeDomainKind.PROVISIONED_SCHEMA_NAME, ExpSchema.DerivationDataScopeType.ParentOnly.name()).getObject(Long.class)); + results.put("sampleTypeParentAndAliquotField", new SqlSelector(schema, """ + SELECT COUNT(DISTINCT D.PropertyURI) FROM + exp.PropertyDescriptor D\s + JOIN exp.PropertyDomain PD ON D.propertyId = PD.propertyid + JOIN exp.DomainDescriptor DD on PD.domainID = DD.domainId + WHERE DD.storageSchemaName = ? AND D.derivationDataScope = ?""", SampleTypeDomainKind.PROVISIONED_SCHEMA_NAME, ExpSchema.DerivationDataScopeType.All.name()).getObject(Long.class)); + + results.put("attachmentColumnCount", new SqlSelector(schema, "SELECT COUNT(*) FROM exp.propertydescriptor WHERE rangeURI = ?", PropertyType.ATTACHMENT.getTypeUri()).getObject(Long.class)); + results.put("dataClassWithAttachmentColumnCount", new SqlSelector(schema, """ + SELECT COUNT(DISTINCT DD.DomainURI) FROM + exp.PropertyDescriptor D\s + JOIN exp.PropertyDomain PD ON D.propertyId = PD.propertyid + JOIN exp.DomainDescriptor DD on PD.domainID = DD.domainId + WHERE DD.storageSchemaName = ? AND D.rangeURI = ?""", DataClassDomainKind.PROVISIONED_SCHEMA_NAME, PropertyType.ATTACHMENT.getTypeUri()).getObject(Long.class)); + results.put("dataClassWithBooleanColumnCount", new SqlSelector(schema, """ + SELECT COUNT(DISTINCT DD.DomainURI) FROM + exp.PropertyDescriptor D\s + JOIN exp.PropertyDomain PD ON D.propertyId = PD.propertyid + JOIN exp.DomainDescriptor DD on PD.domainID = DD.domainId + WHERE DD.storageSchemaName = ? AND D.rangeURI = ?""", DataClassDomainKind.PROVISIONED_SCHEMA_NAME, PropertyType.BOOLEAN.getTypeUri()).getObject(Long.class)); + + results.put("textChoiceColumnCount", new SqlSelector(schema, "SELECT COUNT(*) FROM exp.propertydescriptor WHERE concepturi = ?", TEXT_CHOICE_CONCEPT_URI).getObject(Long.class)); + + results.put("domainsWithDateTimeColumnCount", new SqlSelector(schema, """ + SELECT COUNT(DISTINCT DD.DomainURI) FROM + exp.PropertyDescriptor D\s + JOIN exp.PropertyDomain PD ON D.propertyId = PD.propertyid + JOIN exp.DomainDescriptor DD on PD.domainID = DD.domainId + WHERE D.rangeURI = ?""", PropertyType.DATE_TIME.getTypeUri()).getObject(Long.class)); + + results.put("domainsWithDateColumnCount", new SqlSelector(schema, """ + SELECT COUNT(DISTINCT DD.DomainURI) FROM + exp.PropertyDescriptor D\s + JOIN exp.PropertyDomain PD ON D.propertyId = PD.propertyid + JOIN exp.DomainDescriptor DD on PD.domainID = DD.domainId + WHERE D.rangeURI = ?""", PropertyType.DATE.getTypeUri()).getObject(Long.class)); + + results.put("domainsWithTimeColumnCount", new SqlSelector(schema, """ + SELECT COUNT(DISTINCT DD.DomainURI) FROM + exp.PropertyDescriptor D\s + JOIN exp.PropertyDomain PD ON D.propertyId = PD.propertyid + JOIN exp.DomainDescriptor DD on PD.domainID = DD.domainId + WHERE D.rangeURI = ?""", PropertyType.TIME.getTypeUri()).getObject(Long.class)); + + results.put("maxObjectObjectId", new SqlSelector(schema, "SELECT MAX(ObjectId) FROM exp.Object").getObject(Long.class)); + results.put("maxMaterialRowId", new SqlSelector(schema, "SELECT MAX(RowId) FROM exp.Material").getObject(Long.class)); + + results.putAll(ExperimentService.get().getDomainMetrics()); + + return results; + }); + } + + ExperimentMigrationSchemaHandler handler = new ExperimentMigrationSchemaHandler(); + DatabaseMigrationService.get().registerSchemaHandler(handler); + DatabaseMigrationService.get().registerTableHandler(new MigrationTableHandler() + { + @Override + public TableInfo getTableInfo() + { + return DbSchema.get("premium", DbSchemaType.Bare).getTable("Exclusions"); + } + + @Override + public void adjustFilter(TableInfo sourceTable, SimpleFilter filter, Set containers) + { + // Include experiment runs that were copied + FilterClause includedClause = handler.getIncludedRowIdClause(sourceTable, FieldKey.fromParts("RunId")); + if (includedClause != null) + filter.addClause(includedClause); + } + }); + DatabaseMigrationService.get().registerTableHandler(new MigrationTableHandler() + { + @Override + public TableInfo getTableInfo() + { + return DbSchema.get("premium", DbSchemaType.Bare).getTable("ExclusionMaps"); + } + + @Override + public void adjustFilter(TableInfo sourceTable, SimpleFilter filter, Set containers) + { + // Include experiment runs that were copied + FilterClause includedClause = handler.getIncludedRowIdClause(sourceTable, FieldKey.fromParts("ExclusionId", "RunId")); + if (includedClause != null) + filter.addClause(includedClause); + } + }); + DatabaseMigrationService.get().registerTableHandler(new MigrationTableHandler() + { + @Override + public TableInfo getTableInfo() + { + return DbSchema.get("assayrequest", DbSchemaType.Bare).getTable("RequestRunsJunction"); + } + + @Override + public void adjustFilter(TableInfo sourceTable, SimpleFilter filter, Set containers) + { + // Include experiment runs that were copied + FilterClause includedClause = handler.getIncludedRowIdClause(sourceTable, FieldKey.fromParts("RunId")); + if (includedClause != null) + filter.addClause(includedClause); + } + }); + DatabaseMigrationService.get().registerSchemaHandler(new SampleTypeMigrationSchemaHandler()); + DataClassMigrationSchemaHandler dcHandler = new DataClassMigrationSchemaHandler(); + DatabaseMigrationService.get().registerSchemaHandler(dcHandler); + ExperimentDeleteService.setInstance(dcHandler); + } + + @Override + @NotNull + public Collection getSummary(Container c) + { + Collection list = new LinkedList<>(); + int runGroupCount = ExperimentService.get().getExperiments(c, null, false, true).size(); + if (runGroupCount > 0) + list.add(StringUtilsLabKey.pluralize(runGroupCount, "Run Group")); + + User user = HttpView.currentContext().getUser(); + + Set runTypes = ExperimentService.get().getExperimentRunTypes(c); + for (ExperimentRunType runType : runTypes) + { + if (runType == ExperimentRunType.ALL_RUNS_TYPE) + continue; + + long runCount = runType.getRunCount(user, c); + if (runCount > 0) + list.add(runCount + " runs of type " + runType.getDescription()); + } + + int dataClassCount = ExperimentService.get().getDataClasses(c, null, false).size(); + if (dataClassCount > 0) + list.add(dataClassCount + " Data Class" + (dataClassCount > 1 ? "es" : "")); + + int sampleTypeCount = SampleTypeService.get().getSampleTypes(c, false).size(); + if (sampleTypeCount > 0) + list.add(sampleTypeCount + " Sample Type" + (sampleTypeCount > 1 ? "s" : "")); + + return list; + } + + @Override + public @NotNull ArrayList getDetailedSummary(Container c, User user) + { + ArrayList summaries = new ArrayList<>(); + + // Assay types + long assayTypeCount = AssayService.get().getAssayProtocols(c).stream().filter(p -> p.getContainer().equals(c)).count(); + if (assayTypeCount > 0) + summaries.add(new Summary(assayTypeCount, "Assay Type")); + + // Run count + int runGroupCount = ExperimentService.get().getExperiments(c, user, false, true).size(); + if (runGroupCount > 0) + summaries.add(new Summary(runGroupCount, "Assay run")); + + // Number of Data Classes + List dataClasses = ExperimentService.get().getDataClasses(c, user, false); + int dataClassCount = dataClasses.size(); + if (dataClassCount > 0) + summaries.add(new Summary(dataClassCount, "Data Class")); + + ExpSchema expSchema = new ExpSchema(user, c); + + // Individual Data Class row counts + { + // The table-level container filter is set to ensure data class types are included + // that may not be defined in the target container but may have rows of data in the target container + TableInfo table = ExpSchema.TableType.DataClasses.createTable(expSchema, null, ContainerFilter.Type.CurrentPlusProjectAndShared.create(c, user)); + + // Issue 47919: The "DataCount" column is filtered to only count data in the target container + if (table instanceof ExpDataClassTableImpl tableImpl) + tableImpl.setDataCountContainerFilter(ContainerFilter.Type.Current.create(c, user)); + + Set columns = new LinkedHashSet<>(); + columns.add(ExpDataClassTable.Column.Name.name()); + columns.add(ExpDataClassTable.Column.DataCount.name()); + + Map results = new TableSelector(table, columns).getValueMap(String.class); + for (var entry : results.entrySet()) + { + long count = entry.getValue().longValue(); + if (count > 0) + summaries.add(new Summary(count, entry.getKey())); + } + } + + // Sample Types + int sampleTypeCount = SampleTypeService.get().getSampleTypes(c, false).size(); + if (sampleTypeCount > 0) + summaries.add(new Summary(sampleTypeCount, "Sample Type")); + + // Individual Sample Type row counts + { + // The table-level container filter is set to ensure data class types are included + // that may not be defined in the target container but may have rows of data in the target container + TableInfo table = ExpSchema.TableType.SampleSets.createTable(expSchema, null, ContainerFilter.Type.CurrentPlusProjectAndShared.create(c, user)); + + // Issue 51557: The "SampleCount" column is filtered to only count data in the target container + if (table instanceof ExpSampleTypeTableImpl tableImpl) + tableImpl.setSampleCountContainerFilter(ContainerFilter.Type.Current.create(c, user)); + + Set columns = new LinkedHashSet<>(); + columns.add(ExpSampleTypeTable.Column.Name.name()); + columns.add(ExpSampleTypeTable.Column.SampleCount.name()); + + Map results = new TableSelector(table, columns).getValueMap(String.class); + for (var entry : results.entrySet()) + { + long count = entry.getValue().longValue(); + if (count > 0) + { + String name = entry.getKey(); + Summary s = name.equals("MixtureBatches") + ? new Summary(count, "Batch") + : new Summary(count, name); + summaries.add(s); + } + } + } + + return summaries; + } + + @Override + public @NotNull Set> getIntegrationTests() + { + return Set.of( + DomainImpl.TestCase.class, + DomainPropertyImpl.TestCase.class, + ExpDataTableImpl.TestCase.class, + ExperimentServiceImpl.AuditDomainUriTest.class, + ExperimentServiceImpl.LineageQueryTestCase.class, + ExperimentServiceImpl.ParseInputOutputAliasTestCase.class, + ExperimentServiceImpl.TestCase.class, + ExperimentStressTest.class, + LineagePerfTest.class, + LineageTest.class, + OntologyManager.TestCase.class, + PropertyServiceImpl.TestCase.class, + SampleTypeServiceImpl.TestCase.class, + StorageNameGenerator.TestCase.class, + StorageProvisionerImpl.TestCase.class, + UniqueValueCounterTestCase.class, + XarTestPipelineJob.TestCase.class + ); + } + + @Override + public @NotNull Collection>> getIntegrationTestFactories() + { + List>> list = new ArrayList<>(super.getIntegrationTestFactories()); + list.add(new JspTestCase("/org/labkey/experiment/api/ExpDataClassDataTestCase.jsp")); + list.add(new JspTestCase("/org/labkey/experiment/api/ExpSampleTypeTestCase.jsp")); + return list; + } + + @Override + public @NotNull Set> getUnitTests() + { + return Set.of( + GraphAlgorithms.TestCase.class, + LSIDRelativizer.TestCase.class, + Lsid.TestCase.class, + LsidUtils.TestCase.class, + PropertyController.TestCase.class, + Quantity.TestCase.class, + Unit.TestCase.class + ); + } + + @Override + @NotNull + public Collection getSchemaNames() + { + return List.of( + ExpSchema.SCHEMA_NAME, + DataClassDomainKind.PROVISIONED_SCHEMA_NAME, + SampleTypeDomainKind.PROVISIONED_SCHEMA_NAME + ); + } + + @NotNull + @Override + public Collection getProvisionedSchemaNames() + { + return PageFlowUtil.set(DataClassDomainKind.PROVISIONED_SCHEMA_NAME, SampleTypeDomainKind.PROVISIONED_SCHEMA_NAME); + } +} From dbc018d3dd1e89718dfab6bdeedb3ca2dc7ccdeb Mon Sep 17 00:00:00 2001 From: XingY Date: Fri, 9 Jan 2026 09:29:42 -0800 Subject: [PATCH 06/14] include rowId --- .../org/labkey/api/exp/query/ExpStaleSampleFilesTable.java | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/api/src/org/labkey/api/exp/query/ExpStaleSampleFilesTable.java b/api/src/org/labkey/api/exp/query/ExpStaleSampleFilesTable.java index 67f7dfd9eb9..c0abddbb046 100644 --- a/api/src/org/labkey/api/exp/query/ExpStaleSampleFilesTable.java +++ b/api/src/org/labkey/api/exp/query/ExpStaleSampleFilesTable.java @@ -51,7 +51,7 @@ public FileUnionTable(@NotNull UserSchema schema) .append(materialTable, "m") .append(" ON if.SourceKey = m.RowId"); - SQLFragment staleFileSql = new SQLFragment("SELECT ed.name as filename, ed.container, ed.created, ed.createdBy, ed.DataFileUrl FROM ") + SQLFragment staleFileSql = new SQLFragment("SELECT ed.rowId, ed.name as filename, ed.container, ed.created, ed.createdBy, ed.DataFileUrl FROM ") .append(expDataTable, "ed") .append(" LEFT JOIN (") .append(sampleFileSql) @@ -76,6 +76,11 @@ public FileUnionTable(@NotNull UserSchema schema) addColumn(filePathCol); } + var rowIdCol = new BaseColumnInfo("RowId", this, JdbcType.INTEGER); + rowIdCol.setHidden(true); + rowIdCol.setKeyField(true); + addColumn(rowIdCol); + var containerCol = new BaseColumnInfo("Container", this, JdbcType.VARCHAR); containerCol.setConceptURI(BuiltInColumnTypes.CONTAINERID_CONCEPT_URI); addColumn(containerCol); From e103580f1bc1157427b8a152dc8b62235e83a6a9 Mon Sep 17 00:00:00 2001 From: XingY Date: Fri, 9 Jan 2026 11:31:38 -0800 Subject: [PATCH 07/14] fix sql --- .../src/org/labkey/experiment/api/ExpFilesTableImpl.java | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/experiment/src/org/labkey/experiment/api/ExpFilesTableImpl.java b/experiment/src/org/labkey/experiment/api/ExpFilesTableImpl.java index 2df1a77dc1d..1e19050e6ef 100644 --- a/experiment/src/org/labkey/experiment/api/ExpFilesTableImpl.java +++ b/experiment/src/org/labkey/experiment/api/ExpFilesTableImpl.java @@ -37,6 +37,7 @@ import org.labkey.api.security.UserPrincipal; import org.labkey.api.security.permissions.InsertPermission; import org.labkey.api.security.permissions.Permission; +import org.labkey.api.settings.AppProps; import org.labkey.api.util.InputBuilder; import org.labkey.api.view.ActionURL; import org.labkey.api.writer.HtmlWriter; @@ -46,6 +47,8 @@ import java.util.ArrayList; import java.util.List; +import static org.labkey.api.exp.query.ExpSchema.SAMPLE_FILES_TABLE; + public class ExpFilesTableImpl extends ExpDataTableImpl { protected FileContentService _svc = FileContentService.get(); @@ -117,7 +120,8 @@ protected void populateColumns() ActionURL deleteUrl = ExperimentController.ExperimentUrlsImpl.get().getDeleteDatasURL(getContainer(), null); setDeleteURL(new DetailsURL(deleteUrl)); - addColumn(getFileLinkReferenceCountColumn()); + if (AppProps.getInstance().isOptionalFeatureEnabled(SAMPLE_FILES_TABLE)) + addColumn(getFileLinkReferenceCountColumn()); } public void setDefaultColumns(List customProps) From 8d84c5c73150e944d356124584c4fd0af6e2b93c Mon Sep 17 00:00:00 2001 From: labkey-susanh Date: Fri, 9 Jan 2026 13:18:30 -0800 Subject: [PATCH 08/14] Add RowId as key column to table and move display column to ExpData table --- .../labkey/api/exp/query/ExpDataTable.java | 1 + .../exp/query/ExpStaleSampleFilesTable.java | 14 ++++-- .../experiment/api/ExpDataTableImpl.java | 46 ++++++++++++++++-- .../experiment/api/ExpFilesTableImpl.java | 47 ------------------- 4 files changed, 54 insertions(+), 54 deletions(-) diff --git a/api/src/org/labkey/api/exp/query/ExpDataTable.java b/api/src/org/labkey/api/exp/query/ExpDataTable.java index b60aa86b18c..7169b39a81b 100644 --- a/api/src/org/labkey/api/exp/query/ExpDataTable.java +++ b/api/src/org/labkey/api/exp/query/ExpDataTable.java @@ -35,6 +35,7 @@ enum Column SourceProtocolApplication, SourceApplicationInput, DataFileUrl, + ReferenceCount, Run, RunApplication, RunApplicationOutput, diff --git a/api/src/org/labkey/api/exp/query/ExpStaleSampleFilesTable.java b/api/src/org/labkey/api/exp/query/ExpStaleSampleFilesTable.java index 67f7dfd9eb9..51cdd494387 100644 --- a/api/src/org/labkey/api/exp/query/ExpStaleSampleFilesTable.java +++ b/api/src/org/labkey/api/exp/query/ExpStaleSampleFilesTable.java @@ -51,7 +51,7 @@ public FileUnionTable(@NotNull UserSchema schema) .append(materialTable, "m") .append(" ON if.SourceKey = m.RowId"); - SQLFragment staleFileSql = new SQLFragment("SELECT ed.name as filename, ed.container, ed.created, ed.createdBy, ed.DataFileUrl FROM ") + SQLFragment staleFileSql = new SQLFragment("SELECT ed.rowId, ed.name as filename, ed.container, ed.created, ed.createdBy, ed.DataFileUrl FROM ") .append(expDataTable, "ed") .append(" LEFT JOIN (") .append(sampleFileSql) @@ -65,9 +65,13 @@ public FileUnionTable(@NotNull UserSchema schema) _query.appendComment("", getSchema().getSqlDialect()); + var rowIdCol = new BaseColumnInfo("RowId", this, JdbcType.INTEGER); + rowIdCol.setHidden(true); + rowIdCol.setKeyField(true); + addColumn(rowIdCol); - var filePathShortCol = new BaseColumnInfo("FileName", this, JdbcType.VARCHAR); - addColumn(filePathShortCol); + var fileNameCol = new BaseColumnInfo("FileName", this, JdbcType.VARCHAR); + addColumn(fileNameCol); if (schema.getUser().hasApplicationAdminPermission()) { @@ -87,6 +91,10 @@ public FileUnionTable(@NotNull UserSchema schema) UserIdForeignKey.initColumn(createdByCol); addColumn(createdByCol); +// var referenceCol = new BaseColumnInfo("ReferenceCount", this, JdbcType.INTEGER); +// referenceCol.setHidden(true); +// addColumn(referenceCol); + } @NotNull diff --git a/experiment/src/org/labkey/experiment/api/ExpDataTableImpl.java b/experiment/src/org/labkey/experiment/api/ExpDataTableImpl.java index 42fdec657a2..f7e4ab3b95e 100644 --- a/experiment/src/org/labkey/experiment/api/ExpDataTableImpl.java +++ b/experiment/src/org/labkey/experiment/api/ExpDataTableImpl.java @@ -34,12 +34,14 @@ import org.labkey.api.data.Container; import org.labkey.api.data.ContainerFilter; import org.labkey.api.data.ContainerManager; +import org.labkey.api.data.CoreSchema; import org.labkey.api.data.ExcelWriter; import org.labkey.api.data.JdbcType; import org.labkey.api.data.MutableColumnInfo; import org.labkey.api.data.RenderContext; import org.labkey.api.data.SQLFragment; import org.labkey.api.data.SimpleFilter; +import org.labkey.api.data.SqlSelector; import org.labkey.api.exp.DomainDescriptor; import org.labkey.api.exp.OntologyManager; import org.labkey.api.exp.PropertyColumn; @@ -231,10 +233,46 @@ public List addFileColumns(boolean isFilesTable) private MutableColumnInfo getFileLinkReferenceCountColumn() { - FileLinkFileListener fileListener = new FileLinkFileListener(); - SQLFragment unionSql = fileListener.listFilesQuery(true, new SQLFragment(ExprColumn.STR_TABLE_ALIAS + ".DataFileUrl")); - SQLFragment countSql = new SQLFragment("(SELECT COUNT(*) FROM (").append(unionSql).append("))"); - return new ExprColumn(this, FieldKey.fromParts("ReferenceCount"), countSql, JdbcType.INTEGER); + var result = wrapColumn(Column.ReferenceCount.name(), _rootTable.getColumn("RowId")); + result.setDescription("The number of references to this file from File fields in any domain."); + result.setJdbcType(JdbcType.INTEGER); + result.setDisplayColumnFactory(colInfo -> new ExpDataFileColumn(colInfo) + { + @Override + protected void renderData(HtmlWriter out, ExpData data) + { + + if (data == null || StringUtils.isEmpty(data.getDataFileUrl())) + out.write(""); + else + { + FileLinkFileListener fileListener = new FileLinkFileListener(); + SQLFragment unionSql = fileListener.listFilesQuery(true, data.getFile().getAbsolutePath()); + + long count = new SqlSelector(CoreSchema.getInstance().getSchema(), unionSql).getRowCount(); + + out.write(count); + } + } + + @Override + protected Object getJsonValue(ExpData data) + { + Object val; + if (data == null || StringUtils.isEmpty(data.getDataFileUrl())) + val = null; + else + { + FileLinkFileListener fileListener = new FileLinkFileListener(); + SQLFragment unionSql = fileListener.listFilesQuery(true, data.getFile().getAbsolutePath()); + + val = new SqlSelector(CoreSchema.getInstance().getSchema(), unionSql).getRowCount(); + + } + return val; + } + }); + return result; } @Override diff --git a/experiment/src/org/labkey/experiment/api/ExpFilesTableImpl.java b/experiment/src/org/labkey/experiment/api/ExpFilesTableImpl.java index 2df1a77dc1d..92fd2938a65 100644 --- a/experiment/src/org/labkey/experiment/api/ExpFilesTableImpl.java +++ b/experiment/src/org/labkey/experiment/api/ExpFilesTableImpl.java @@ -117,7 +117,6 @@ protected void populateColumns() ActionURL deleteUrl = ExperimentController.ExperimentUrlsImpl.get().getDeleteDatasURL(getContainer(), null); setDeleteURL(new DetailsURL(deleteUrl)); - addColumn(getFileLinkReferenceCountColumn()); } public void setDefaultColumns(List customProps) @@ -228,52 +227,6 @@ protected Object getJsonValue(ExpData data) return result; } - private MutableColumnInfo getFileLinkReferenceCountColumn() - { -// FileLinkFileListener fileListener = new FileLinkFileListener(); -// SQLFragment unionSql = fileListener.listFilesQuery(true, new SQLFragment(ExprColumn.STR_TABLE_ALIAS + ".DataFileUrl")); -// return new ExprColumn(this, FieldKey.fromParts("ReferenceCount"), unionSql, JdbcType.INTEGER); - var result = wrapColumn("ReferenceCountDisplay", _rootTable.getColumn("RowId")); - result.setJdbcType(JdbcType.VARCHAR); - result.setDisplayColumnFactory(colInfo -> new ExpDataFileColumn(colInfo) - { - @Override - protected void renderData(HtmlWriter out, ExpData data) - { - - if (data == null || StringUtils.isEmpty(data.getDataFileUrl())) - out.write(""); - else - { - FileLinkFileListener fileListener = new FileLinkFileListener(); - SQLFragment unionSql = fileListener.listFilesQuery(true, data.getFile().getAbsolutePath()); - - long count = new SqlSelector(CoreSchema.getInstance().getSchema(), unionSql).getRowCount(); - - out.write(count); - } - } - - @Override - protected Object getJsonValue(ExpData data) - { - Object val; - if (data == null || StringUtils.isEmpty(data.getDataFileUrl())) - val = null; - else - { - FileLinkFileListener fileListener = new FileLinkFileListener(); - SQLFragment unionSql = fileListener.listFilesQuery(true, data.getFile().getAbsolutePath()); - - val = new SqlSelector(CoreSchema.getInstance().getSchema(), unionSql).getRowCount(); - - } - return val; - } - }); - return result; - } - @Override public boolean hasPermission(@NotNull UserPrincipal user, @NotNull Class perm) { From 6bec4b56862e3164940f5274486018fd47b22f13 Mon Sep 17 00:00:00 2001 From: labkey-susanh Date: Fri, 9 Jan 2026 13:58:09 -0800 Subject: [PATCH 09/14] Export experimental feature flag to moduleContext --- .../org/labkey/experiment/ExperimentModule.java | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/experiment/src/org/labkey/experiment/ExperimentModule.java b/experiment/src/org/labkey/experiment/ExperimentModule.java index 435448ca0b7..bd94e482040 100644 --- a/experiment/src/org/labkey/experiment/ExperimentModule.java +++ b/experiment/src/org/labkey/experiment/ExperimentModule.java @@ -18,6 +18,7 @@ import org.apache.commons.lang3.math.NumberUtils; import org.jetbrains.annotations.NotNull; import org.jetbrains.annotations.Nullable; +import org.json.JSONObject; import org.labkey.api.admin.FolderSerializationRegistry; import org.labkey.api.assay.AssayProvider; import org.labkey.api.assay.AssayService; @@ -112,6 +113,7 @@ import org.labkey.api.vocabulary.security.DesignVocabularyPermission; import org.labkey.api.webdav.WebdavResource; import org.labkey.api.webdav.WebdavService; +import org.labkey.api.writer.ContainerUser; import org.labkey.experiment.api.DataClassDomainKind; import org.labkey.experiment.api.ExpDataClassImpl; import org.labkey.experiment.api.ExpDataClassTableImpl; @@ -181,6 +183,7 @@ import static org.labkey.api.data.ColumnRenderPropertiesImpl.STORAGE_UNIQUE_ID_CONCEPT_URI; import static org.labkey.api.data.ColumnRenderPropertiesImpl.TEXT_CHOICE_CONCEPT_URI; import static org.labkey.api.exp.api.ExperimentService.MODULE_NAME; +import static org.labkey.api.exp.query.ExpSchema.SAMPLE_FILES_TABLE; public class ExperimentModule extends SpringModule { @@ -266,6 +269,9 @@ protected void init() } else { + OptionalFeatureService.get().addExperimentalFeatureFlag(SAMPLE_FILES_TABLE, "Manage Unreferenced Sample Files", + "Enable 'Unreferenced Sample Files' table to view and delete sample files that are no longer referenced by samples", false); + OptionalFeatureService.get().addExperimentalFeatureFlag(NameGenerator.EXPERIMENTAL_ALLOW_GAP_COUNTER, "Allow gap with withCounter and rootSampleCount expression", "Check this option if gaps in the count generated by withCounter or rootSampleCount name expression are allowed.", true); } @@ -1117,4 +1123,12 @@ public Collection getProvisionedSchemaNames() { return PageFlowUtil.set(DataClassDomainKind.PROVISIONED_SCHEMA_NAME, SampleTypeDomainKind.PROVISIONED_SCHEMA_NAME); } + + @Override + public JSONObject getPageContextJson(ContainerUser context) + { + JSONObject json = new JSONObject(getDefaultPageContextJson(context.getContainer())); + json.put(SAMPLE_FILES_TABLE, OptionalFeatureService.get().isFeatureEnabled(SAMPLE_FILES_TABLE)); + return json; + } } From bdb88934217755b06232cce82569a28b694e9773 Mon Sep 17 00:00:00 2001 From: labkey-susanh Date: Fri, 9 Jan 2026 15:46:23 -0800 Subject: [PATCH 10/14] Hide column and exclude column when feature is not enabled --- .../exp/query/ExpUnreferencedSampleFilesTable.java | 13 ++++++++----- .../org/labkey/experiment/api/ExpDataTableImpl.java | 5 ++++- 2 files changed, 12 insertions(+), 6 deletions(-) diff --git a/api/src/org/labkey/api/exp/query/ExpUnreferencedSampleFilesTable.java b/api/src/org/labkey/api/exp/query/ExpUnreferencedSampleFilesTable.java index 18b20d562ab..29fd877b87c 100644 --- a/api/src/org/labkey/api/exp/query/ExpUnreferencedSampleFilesTable.java +++ b/api/src/org/labkey/api/exp/query/ExpUnreferencedSampleFilesTable.java @@ -1,5 +1,6 @@ package org.labkey.api.exp.query; +import org.apache.commons.lang3.StringUtils; import org.jetbrains.annotations.NotNull; import org.labkey.api.data.BaseColumnInfo; import org.labkey.api.data.ContainerFilter; @@ -38,12 +39,19 @@ public FileUnionTable(@NotNull UserSchema schema) super(CoreSchema.getInstance().getSchema(), ExpSchema.SAMPLE_FILES_TABLE, schema); FileContentService svc = FileContentService.get(); + _query = new SQLFragment(); + if (svc == null) + return; _query.appendComment("", getSchema().getSqlDialect()); TableInfo expDataTable = ExperimentService.get().getTinfoData(); TableInfo materialTable = ExperimentService.get().getTinfoMaterial(); + SQLFragment listQuery = svc.listSampleFilesQuery(schema.getUser()); + if (StringUtils.isEmpty(listQuery)) + return; + SQLFragment sampleFileSql = new SQLFragment("SELECT m.Container, if.FilePathShort \n") .append("FROM (") .append(svc.listSampleFilesQuery(schema.getUser())) @@ -91,11 +99,6 @@ public FileUnionTable(@NotNull UserSchema schema) var createdByCol = new BaseColumnInfo("CreatedBy", this, JdbcType.INTEGER); UserIdForeignKey.initColumn(createdByCol); addColumn(createdByCol); - -// var referenceCol = new BaseColumnInfo("ReferenceCount", this, JdbcType.INTEGER); -// referenceCol.setHidden(true); -// addColumn(referenceCol); - } @NotNull diff --git a/experiment/src/org/labkey/experiment/api/ExpDataTableImpl.java b/experiment/src/org/labkey/experiment/api/ExpDataTableImpl.java index f7e4ab3b95e..bdf7a498ecf 100644 --- a/experiment/src/org/labkey/experiment/api/ExpDataTableImpl.java +++ b/experiment/src/org/labkey/experiment/api/ExpDataTableImpl.java @@ -77,6 +77,7 @@ import org.labkey.api.security.User; import org.labkey.api.security.permissions.InsertPermission; import org.labkey.api.security.permissions.UpdatePermission; +import org.labkey.api.settings.AppProps; import org.labkey.api.util.ConfigurationException; import org.labkey.api.util.FileUtil; import org.labkey.api.util.HtmlString; @@ -203,7 +204,8 @@ public List addFileColumns(boolean isFilesTable) addColumn(Column.FileExtension); addColumn(Column.WebDavUrl); addColumn(Column.WebDavUrlRelative); - addColumn(getFileLinkReferenceCountColumn()); + if (AppProps.getInstance().isOptionalFeatureEnabled(ExpSchema.SAMPLE_FILES_TABLE)) + addColumn(getFileLinkReferenceCountColumn()); var flagCol = addColumn(Column.Flag); if (isFilesTable) flagCol.setLabel("Description"); @@ -236,6 +238,7 @@ private MutableColumnInfo getFileLinkReferenceCountColumn() var result = wrapColumn(Column.ReferenceCount.name(), _rootTable.getColumn("RowId")); result.setDescription("The number of references to this file from File fields in any domain."); result.setJdbcType(JdbcType.INTEGER); + result.setHidden(true); result.setDisplayColumnFactory(colInfo -> new ExpDataFileColumn(colInfo) { @Override From 62ac8de2641f3df87e0878ce68f006e65441a20b Mon Sep 17 00:00:00 2001 From: labkey-susanh Date: Mon, 12 Jan 2026 13:26:08 -0800 Subject: [PATCH 11/14] Add comment --- experiment/src/org/labkey/experiment/api/ExpDataTableImpl.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/experiment/src/org/labkey/experiment/api/ExpDataTableImpl.java b/experiment/src/org/labkey/experiment/api/ExpDataTableImpl.java index bdf7a498ecf..a7df3aa3285 100644 --- a/experiment/src/org/labkey/experiment/api/ExpDataTableImpl.java +++ b/experiment/src/org/labkey/experiment/api/ExpDataTableImpl.java @@ -233,6 +233,8 @@ public List addFileColumns(boolean isFilesTable) return customProps; } + // This is included in exp.data, not just exp.files because we want to be able to show a filtered view of + // sample files from our applications, and exp.files will not show subfolders private MutableColumnInfo getFileLinkReferenceCountColumn() { var result = wrapColumn(Column.ReferenceCount.name(), _rootTable.getColumn("RowId")); From 2a78a9d23b297e15520f8e662e81d1f133c266f0 Mon Sep 17 00:00:00 2001 From: labkey-susanh Date: Tue, 13 Jan 2026 06:43:18 -0800 Subject: [PATCH 12/14] Refactor for better code --- .../experiment/api/ExpDataTableImpl.java | 33 ++++++++----------- 1 file changed, 14 insertions(+), 19 deletions(-) diff --git a/experiment/src/org/labkey/experiment/api/ExpDataTableImpl.java b/experiment/src/org/labkey/experiment/api/ExpDataTableImpl.java index a7df3aa3285..ce2db439e6e 100644 --- a/experiment/src/org/labkey/experiment/api/ExpDataTableImpl.java +++ b/experiment/src/org/labkey/experiment/api/ExpDataTableImpl.java @@ -243,38 +243,33 @@ private MutableColumnInfo getFileLinkReferenceCountColumn() result.setHidden(true); result.setDisplayColumnFactory(colInfo -> new ExpDataFileColumn(colInfo) { - @Override - protected void renderData(HtmlWriter out, ExpData data) + private Long getCount(ExpData data) { - - if (data == null || StringUtils.isEmpty(data.getDataFileUrl())) - out.write(""); + if (data == null || StringUtils.isEmpty(data.getDataFileUrl()) || data.getFile() == null) + return null; else { FileLinkFileListener fileListener = new FileLinkFileListener(); SQLFragment unionSql = fileListener.listFilesQuery(true, data.getFile().getAbsolutePath()); - long count = new SqlSelector(CoreSchema.getInstance().getSchema(), unionSql).getRowCount(); - - out.write(count); + return new SqlSelector(CoreSchema.getInstance().getSchema(), unionSql).getRowCount(); } } @Override - protected Object getJsonValue(ExpData data) + protected void renderData(HtmlWriter out, ExpData data) { - Object val; - if (data == null || StringUtils.isEmpty(data.getDataFileUrl())) - val = null; + Long val = getCount(data); + if (val == null) + out.write(""); else - { - FileLinkFileListener fileListener = new FileLinkFileListener(); - SQLFragment unionSql = fileListener.listFilesQuery(true, data.getFile().getAbsolutePath()); - - val = new SqlSelector(CoreSchema.getInstance().getSchema(), unionSql).getRowCount(); + out.write(val); + } - } - return val; + @Override + protected Object getJsonValue(ExpData data) + { + return getCount(data); } }); return result; From 4b7da07ceb21f4633e333662ce852936c377f64b Mon Sep 17 00:00:00 2001 From: labkey-susanh Date: Tue, 13 Jan 2026 07:02:54 -0800 Subject: [PATCH 13/14] Fix CreatedBy foreign key --- .../labkey/api/exp/query/ExpUnreferencedSampleFilesTable.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/api/src/org/labkey/api/exp/query/ExpUnreferencedSampleFilesTable.java b/api/src/org/labkey/api/exp/query/ExpUnreferencedSampleFilesTable.java index 29fd877b87c..02bd10e5e88 100644 --- a/api/src/org/labkey/api/exp/query/ExpUnreferencedSampleFilesTable.java +++ b/api/src/org/labkey/api/exp/query/ExpUnreferencedSampleFilesTable.java @@ -12,7 +12,7 @@ import org.labkey.api.exp.api.ExperimentService; import org.labkey.api.files.FileContentService; import org.labkey.api.query.FilteredTable; -import org.labkey.api.query.UserIdForeignKey; +import org.labkey.api.query.UserIdQueryForeignKey; import org.labkey.api.query.UserSchema; import org.labkey.api.query.column.BuiltInColumnTypes; @@ -97,7 +97,7 @@ public FileUnionTable(@NotNull UserSchema schema) addColumn(createdCol); var createdByCol = new BaseColumnInfo("CreatedBy", this, JdbcType.INTEGER); - UserIdForeignKey.initColumn(createdByCol); + createdByCol.setFk(new UserIdQueryForeignKey(getUserSchema(), true)); addColumn(createdByCol); } From 347f319f8e2d759f9ac81a4bb737ea4c370d8cfd Mon Sep 17 00:00:00 2001 From: labkey-susanh Date: Tue, 13 Jan 2026 12:59:39 -0800 Subject: [PATCH 14/14] Minor updates from code review --- .../exp/query/ExpUnreferencedSampleFilesTable.java | 13 +++++++------ api/src/org/labkey/api/files/FileListener.java | 2 +- .../src/org/labkey/experiment/ExperimentModule.java | 2 +- .../org/labkey/experiment/FileLinkFileListener.java | 2 +- .../labkey/experiment/api/ExpFilesTableImpl.java | 1 - .../labkey/filecontent/FileContentServiceImpl.java | 4 ++-- 6 files changed, 12 insertions(+), 12 deletions(-) diff --git a/api/src/org/labkey/api/exp/query/ExpUnreferencedSampleFilesTable.java b/api/src/org/labkey/api/exp/query/ExpUnreferencedSampleFilesTable.java index 02bd10e5e88..9409802b054 100644 --- a/api/src/org/labkey/api/exp/query/ExpUnreferencedSampleFilesTable.java +++ b/api/src/org/labkey/api/exp/query/ExpUnreferencedSampleFilesTable.java @@ -30,11 +30,11 @@ private static TableInfo createVirtualTable(@NotNull ExpSchema schema) return new ExpUnreferencedSampleFilesTable.FileUnionTable(schema); } - private static class FileUnionTable extends VirtualTable + private static class FileUnionTable extends VirtualTable { private final SQLFragment _query; - public FileUnionTable(@NotNull UserSchema schema) + public FileUnionTable(@NotNull ExpSchema schema) { super(CoreSchema.getInstance().getSchema(), ExpSchema.SAMPLE_FILES_TABLE, schema); @@ -43,15 +43,16 @@ public FileUnionTable(@NotNull UserSchema schema) _query = new SQLFragment(); if (svc == null) return; - _query.appendComment("", getSchema().getSqlDialect()); - - TableInfo expDataTable = ExperimentService.get().getTinfoData(); - TableInfo materialTable = ExperimentService.get().getTinfoMaterial(); SQLFragment listQuery = svc.listSampleFilesQuery(schema.getUser()); if (StringUtils.isEmpty(listQuery)) return; + TableInfo expDataTable = ExperimentService.get().getTinfoData(); + TableInfo materialTable = ExperimentService.get().getTinfoMaterial(); + + _query.appendComment("", getSchema().getSqlDialect()); + SQLFragment sampleFileSql = new SQLFragment("SELECT m.Container, if.FilePathShort \n") .append("FROM (") .append(svc.listSampleFilesQuery(schema.getUser())) diff --git a/api/src/org/labkey/api/files/FileListener.java b/api/src/org/labkey/api/files/FileListener.java index 3e5d5810b7c..91702ed4b94 100644 --- a/api/src/org/labkey/api/files/FileListener.java +++ b/api/src/org/labkey/api/files/FileListener.java @@ -95,7 +95,7 @@ default void fileDeleted(@NotNull Path deleted, @Nullable User user, @Nullable C */ SQLFragment listFilesQuery(); - default SQLFragment listSampleFilesQuery() + @Nullable default SQLFragment listSampleFilesQuery() { return null; } diff --git a/experiment/src/org/labkey/experiment/ExperimentModule.java b/experiment/src/org/labkey/experiment/ExperimentModule.java index bd94e482040..90312720f61 100644 --- a/experiment/src/org/labkey/experiment/ExperimentModule.java +++ b/experiment/src/org/labkey/experiment/ExperimentModule.java @@ -1127,7 +1127,7 @@ public Collection getProvisionedSchemaNames() @Override public JSONObject getPageContextJson(ContainerUser context) { - JSONObject json = new JSONObject(getDefaultPageContextJson(context.getContainer())); + JSONObject json = super.getPageContextJson(context); json.put(SAMPLE_FILES_TABLE, OptionalFeatureService.get().isFeatureEnabled(SAMPLE_FILES_TABLE)); return json; } diff --git a/experiment/src/org/labkey/experiment/FileLinkFileListener.java b/experiment/src/org/labkey/experiment/FileLinkFileListener.java index 2b93db1a588..83b1e7ff358 100644 --- a/experiment/src/org/labkey/experiment/FileLinkFileListener.java +++ b/experiment/src/org/labkey/experiment/FileLinkFileListener.java @@ -326,7 +326,7 @@ public SQLFragment listSampleFilesQuery() final SQLFragment frag = new SQLFragment(); hardTableFileLinkColumns((schema, table, pathColumn, containerId, domainUri) -> { - if (schema.getName().equals("expsampleset")) + if (PROVISIONED_SCHEMA_NAME.equalsIgnoreCase(schema.getName())) { SQLFragment containerFrag = new SQLFragment("?", containerId); TableUpdaterFileListener updater = new TableUpdaterFileListener(table, pathColumn.getColumnName(), TableUpdaterFileListener.Type.filePath, "rowid", containerFrag); diff --git a/experiment/src/org/labkey/experiment/api/ExpFilesTableImpl.java b/experiment/src/org/labkey/experiment/api/ExpFilesTableImpl.java index ff5cc09efb3..db6db57f9b7 100644 --- a/experiment/src/org/labkey/experiment/api/ExpFilesTableImpl.java +++ b/experiment/src/org/labkey/experiment/api/ExpFilesTableImpl.java @@ -112,7 +112,6 @@ protected void populateColumns() getMutableColumn(Column.Name).setURL(detailsURL); ActionURL deleteUrl = ExperimentController.ExperimentUrlsImpl.get().getDeleteDatasURL(getContainer(), null); setDeleteURL(new DetailsURL(deleteUrl)); - } public void setDefaultColumns(List customProps) diff --git a/filecontent/src/org/labkey/filecontent/FileContentServiceImpl.java b/filecontent/src/org/labkey/filecontent/FileContentServiceImpl.java index b7010a37e6c..d890353d832 100644 --- a/filecontent/src/org/labkey/filecontent/FileContentServiceImpl.java +++ b/filecontent/src/org/labkey/filecontent/FileContentServiceImpl.java @@ -1242,7 +1242,7 @@ public SQLFragment listSampleFilesQuery(@NotNull User currentUser) for (FileListener fileListener : _fileListeners) { SQLFragment subselect = fileListener.listSampleFilesQuery(); - if (subselect != null) + if (subselect != null && !subselect.isEmpty()) { frag.append(union); frag.append(subselect); @@ -1250,7 +1250,7 @@ public SQLFragment listSampleFilesQuery(@NotNull User currentUser) } } frag.append(")"); - return frag; + return union.isEmpty() ? new SQLFragment() : frag; } @Override