From 685f9ede067cb7edb9049785d8f2b8281c0d25ca Mon Sep 17 00:00:00 2001 From: XingY Date: Thu, 8 Jan 2026 12:39:48 -0800 Subject: [PATCH 1/6] Multi choices --- .../api/gwt/client/model/GWTDomain.java | 672 ++--- .../api/data/ColumnRenderPropertiesImpl.java | 1 + .../org/labkey/api/data/ConvertHelper.java | 2 + api/src/org/labkey/api/data/JdbcType.java | 33 +- api/src/org/labkey/api/data/MultiChoice.java | 29 + .../labkey/api/data/PropertyStorageSpec.java | 1 + .../data/dialect/BasePostgreSqlDialect.java | 7 + .../org/labkey/api/exp/ObjectProperty.java | 3 + api/src/org/labkey/api/exp/PropertyType.java | 2445 +++++++++-------- .../api/exp/api/SampleTypeDomainKind.java | 6 + .../labkey/api/exp/property/DomainKind.java | 878 +++--- .../labkey/api/exp/property/DomainUtil.java | 14 + api/src/org/labkey/api/reader/TabLoader.java | 15 +- api/src/org/labkey/api/settings/AppProps.java | 535 ++-- .../labkey/experiment/ExperimentModule.java | 2259 +++++++-------- .../experiment/api/DataClassDomainKind.java | 6 + .../org/labkey/list/model/ListDomainKind.java | 1494 +++++----- .../labkey/study/model/DatasetDomainKind.java | 1738 ++++++------ 18 files changed, 5136 insertions(+), 5002 deletions(-) diff --git a/api/gwtsrc/org/labkey/api/gwt/client/model/GWTDomain.java b/api/gwtsrc/org/labkey/api/gwt/client/model/GWTDomain.java index eecd41589ee..2550585c35e 100644 --- a/api/gwtsrc/org/labkey/api/gwt/client/model/GWTDomain.java +++ b/api/gwtsrc/org/labkey/api/gwt/client/model/GWTDomain.java @@ -1,335 +1,337 @@ -/* - * Copyright (c) 2018-2019 LabKey Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.labkey.api.gwt.client.model; - -import com.fasterxml.jackson.annotation.JsonIgnore; -import com.google.gwt.user.client.rpc.IsSerializable; -import lombok.Getter; -import lombok.Setter; -import org.labkey.api.gwt.client.DefaultValueType; -import org.labkey.api.gwt.client.util.PropertyUtil; - -import java.util.ArrayList; -import java.util.Collections; -import java.util.HashSet; -import java.util.List; -import java.util.Set; - -public class GWTDomain implements IsSerializable -{ - private String _ts; - @Getter @Setter private int domainId; - @Getter @Setter private String name; - @Getter @Setter private String domainURI; - @Getter @Setter private String domainKindName; - @Getter @Setter private String description; - @Getter @Setter private String container; - @Getter @Setter private boolean allowFileLinkProperties; - @Getter @Setter private boolean allowAttachmentProperties; - @Getter @Setter private boolean allowFlagProperties; - @Getter @Setter private boolean allowTextChoiceProperties; - @Getter @Setter private boolean allowSampleSubjectProperties; - @Getter @Setter private boolean allowTimepointProperties; - @Getter @Setter private boolean allowUniqueConstraintProperties; - @Getter @Setter private boolean allowCalculatedFields; - @Getter @Setter private boolean showDefaultValueSettings; - private DefaultValueType defaultDefaultValueType = null; - private DefaultValueType[] defaultValueOptions = new DefaultValueType[0]; - private List fields = new ArrayList<>(); - private List standardFields = null; - private List calculatedFields = null; - @Getter @Setter private List indices = new ArrayList<>(); - private String defaultValuesURL = null; - private Set mandatoryPropertyDescriptorNames = new HashSet<>(); - private Set reservedFieldNames = new HashSet<>(); - private Set reservedFieldNamePrefixes = new HashSet<>(); - private Set phiNotAllowedFieldNames = new HashSet<>(); - private Set excludeFromExportFieldNames = new HashSet<>(); - @Getter @Setter private boolean provisioned = false; - @Getter @Setter private List disabledSystemFields; - - // schema,query,template are not part of the domain, but it's handy to pass - // these values to the PropertiedEditor along with the GWTDomain. - // NOTE queryName is not necessarily == name - @Getter @Setter private String schemaName = null; - @Getter @Setter private String queryName = null; - @Getter @Setter private String templateDescription = null; // null if no template - @Getter @Setter private String instructions = null; - @Getter @Setter private boolean supportsPhiLevel = false; - - public GWTDomain() - { - } - - // deep clone constructor - public GWTDomain(GWTDomain src) - { - _ts = src._ts; - this.domainId = src.domainId; - this.name = src.name; - this.domainURI = src.domainURI; - this.domainKindName = src.domainKindName; - this.description = src.description; - this.disabledSystemFields = src.disabledSystemFields; - this.container = src.container; - this.allowFileLinkProperties = src.allowFileLinkProperties; - this.allowAttachmentProperties = src.allowAttachmentProperties; - this.allowFlagProperties = src.allowFlagProperties; - this.allowTextChoiceProperties = src.allowTextChoiceProperties; - this.allowSampleSubjectProperties = src.allowSampleSubjectProperties; - this.allowTimepointProperties = src.allowTimepointProperties; - this.allowUniqueConstraintProperties = src.allowUniqueConstraintProperties; - this.allowCalculatedFields = src.allowCalculatedFields; - this.showDefaultValueSettings = src.showDefaultValueSettings; - this.defaultDefaultValueType = src.defaultDefaultValueType; - this.defaultValueOptions = src.defaultValueOptions; - this.defaultValuesURL = src.defaultValuesURL; - this.provisioned = src.provisioned; - this.supportsPhiLevel = src.supportsPhiLevel; - - if (src.indices != null) - { - for (int i = 0; i < src.indices.size(); i++) - this.indices.add(src.indices.get(i).copy()); - } - - // include all fields here (standard and calculated) in the copy - if (src.getFields(true) == null) - return; - for (int i=0 ; i getFields(boolean includeCalculated) - { - if (includeCalculated) - return fields; - else - return getFields(); - } - - public List getFields() - { - if (standardFields == null) - standardFields = fields.stream().filter(f -> f.getValueExpression() == null).toList(); - return standardFields; - } - - public List getCalculatedFields() - { - if (calculatedFields == null) - calculatedFields = fields.stream().filter(f -> f.getValueExpression() != null).toList(); - return calculatedFields; - } - - public void setFields(List list) - { - fields = list; - - // reset the cached lists of fields so they will be recalculated on next call to getters - standardFields = null; - calculatedFields = null; - } - - public FieldType getFieldByName(String name) - { - for (FieldType field : getFields(true)) - { - if (field.getName() != null && field.getName().equalsIgnoreCase(name)) - return field; - } - return null; - } - - /** - * @return Indicates that the property can't be removed from the domain. The property may or may not be nullable. - */ - public boolean isMandatoryField(FieldType field) - { - if (mandatoryPropertyDescriptorNames == null || field.getName() == null) - { - return false; - } - return mandatoryPropertyDescriptorNames.contains(field.getName().toLowerCase()); - } - - public boolean isEditable(FieldType field) - { - return true; - } - - /** - * @return Indicates that the property is not allowed to be set as PHI - */ - public boolean allowsPhi(FieldType field) - { - return !(getPhiNotAllowedFieldNames() != null && field.getName() != null && getPhiNotAllowedFieldNames().contains(field.getName().toLowerCase())); - } - - /** - * @param mandatoryFieldNames names of property descriptors that must be present in this domain. Does not indicate that they must be non-nullable. - */ - public void setMandatoryFieldNames(Set mandatoryFieldNames) - { - this.mandatoryPropertyDescriptorNames = new HashSet<>(); - for (String mandatoryPropertyDescriptor : mandatoryFieldNames) - { - this.mandatoryPropertyDescriptorNames.add(mandatoryPropertyDescriptor.toLowerCase()); - } - } - - /** - * Get the list of property names that can't be removed from the domain. The set of mandatory fields is not modifiable in the designer. - */ - public Set getMandatoryFieldNames() - { - if (this.mandatoryPropertyDescriptorNames == null) - return Collections.emptySet(); - return Collections.unmodifiableSet(this.mandatoryPropertyDescriptorNames); - } - - public Set getReservedFieldNames() - { - return reservedFieldNames; - } - - /** - * @param reservedFieldNames can't create new fields with these names - */ - public void setReservedFieldNames(Set reservedFieldNames) - { - this.reservedFieldNames = new HashSet<>(); - for (String s : reservedFieldNames) - { - this.reservedFieldNames.add(s.toLowerCase()); - } - } - - public Set getReservedFieldNamePrefixes() - { - return this.reservedFieldNamePrefixes; - } - - public void setReservedFieldNamePrefixes(Set prefixes) - { - this.reservedFieldNamePrefixes = new HashSet<>(prefixes); - } - /** - * - * @param excludeFromExportFieldNames These fields will be suppressed from the export field list. Primary use case is to not export List key fields. - */ - public void setExcludeFromExportFieldNames(Set excludeFromExportFieldNames) - { - this.excludeFromExportFieldNames = new HashSet<>(); - for (String excludeFromExportFieldName : excludeFromExportFieldNames) - { - this.excludeFromExportFieldNames.add(excludeFromExportFieldName.toLowerCase()); - } - } - - public Set getExcludeFromExportFieldNames() - { - return excludeFromExportFieldNames; - } - - public boolean isExcludeFromExportField(FieldType field) - { - if (excludeFromExportFieldNames == null || field.getName() == null) - { - return false; - } - return excludeFromExportFieldNames.contains(field.getName().toLowerCase()); - } - - public Set getPhiNotAllowedFieldNames() - { - return phiNotAllowedFieldNames; - } - - public void setPhiNotAllowedFieldNames(Set phiNotAllowedFieldNames) - { - this.phiNotAllowedFieldNames = new HashSet<>(); - for (String fieldName : phiNotAllowedFieldNames) - { - this.phiNotAllowedFieldNames.add(fieldName.toLowerCase()); - } - } - - public DefaultValueType getDefaultDefaultValueType() - { - return defaultDefaultValueType; - } - - public DefaultValueType[] getDefaultValueOptions() - { - return defaultValueOptions; - } - - public void setDefaultValueOptions(DefaultValueType[] defaultOptions, DefaultValueType defaultDefault) - { - this.defaultDefaultValueType = defaultDefault; - this.defaultValueOptions = defaultOptions; - } - - public String getDefaultValuesURL() - { - if (defaultValuesURL == null) - return PropertyUtil.getRelativeURL("setDefaultValuesList", "list"); - return defaultValuesURL; - } - - public void setDefaultValuesURL(String defaultValuesURL) - { - this.defaultValuesURL = defaultValuesURL; - } -} +/* + * Copyright (c) 2018-2019 LabKey Corporation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.labkey.api.gwt.client.model; + +import com.fasterxml.jackson.annotation.JsonIgnore; +import com.google.gwt.user.client.rpc.IsSerializable; +import lombok.Getter; +import lombok.Setter; +import org.labkey.api.gwt.client.DefaultValueType; +import org.labkey.api.gwt.client.util.PropertyUtil; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashSet; +import java.util.List; +import java.util.Set; + +public class GWTDomain implements IsSerializable +{ + private String _ts; + @Getter @Setter private int domainId; + @Getter @Setter private String name; + @Getter @Setter private String domainURI; + @Getter @Setter private String domainKindName; + @Getter @Setter private String description; + @Getter @Setter private String container; + @Getter @Setter private boolean allowFileLinkProperties; + @Getter @Setter private boolean allowAttachmentProperties; + @Getter @Setter private boolean allowFlagProperties; + @Getter @Setter private boolean allowTextChoiceProperties; + @Getter @Setter private boolean allowMultiChoiceProperties; + @Getter @Setter private boolean allowSampleSubjectProperties; + @Getter @Setter private boolean allowTimepointProperties; + @Getter @Setter private boolean allowUniqueConstraintProperties; + @Getter @Setter private boolean allowCalculatedFields; + @Getter @Setter private boolean showDefaultValueSettings; + private DefaultValueType defaultDefaultValueType = null; + private DefaultValueType[] defaultValueOptions = new DefaultValueType[0]; + private List fields = new ArrayList<>(); + private List standardFields = null; + private List calculatedFields = null; + @Getter @Setter private List indices = new ArrayList<>(); + private String defaultValuesURL = null; + private Set mandatoryPropertyDescriptorNames = new HashSet<>(); + private Set reservedFieldNames = new HashSet<>(); + private Set reservedFieldNamePrefixes = new HashSet<>(); + private Set phiNotAllowedFieldNames = new HashSet<>(); + private Set excludeFromExportFieldNames = new HashSet<>(); + @Getter @Setter private boolean provisioned = false; + @Getter @Setter private List disabledSystemFields; + + // schema,query,template are not part of the domain, but it's handy to pass + // these values to the PropertiedEditor along with the GWTDomain. + // NOTE queryName is not necessarily == name + @Getter @Setter private String schemaName = null; + @Getter @Setter private String queryName = null; + @Getter @Setter private String templateDescription = null; // null if no template + @Getter @Setter private String instructions = null; + @Getter @Setter private boolean supportsPhiLevel = false; + + public GWTDomain() + { + } + + // deep clone constructor + public GWTDomain(GWTDomain src) + { + _ts = src._ts; + this.domainId = src.domainId; + this.name = src.name; + this.domainURI = src.domainURI; + this.domainKindName = src.domainKindName; + this.description = src.description; + this.disabledSystemFields = src.disabledSystemFields; + this.container = src.container; + this.allowFileLinkProperties = src.allowFileLinkProperties; + this.allowAttachmentProperties = src.allowAttachmentProperties; + this.allowFlagProperties = src.allowFlagProperties; + this.allowTextChoiceProperties = src.allowTextChoiceProperties; + this.allowMultiChoiceProperties = src.allowMultiChoiceProperties; + this.allowSampleSubjectProperties = src.allowSampleSubjectProperties; + this.allowTimepointProperties = src.allowTimepointProperties; + this.allowUniqueConstraintProperties = src.allowUniqueConstraintProperties; + this.allowCalculatedFields = src.allowCalculatedFields; + this.showDefaultValueSettings = src.showDefaultValueSettings; + this.defaultDefaultValueType = src.defaultDefaultValueType; + this.defaultValueOptions = src.defaultValueOptions; + this.defaultValuesURL = src.defaultValuesURL; + this.provisioned = src.provisioned; + this.supportsPhiLevel = src.supportsPhiLevel; + + if (src.indices != null) + { + for (int i = 0; i < src.indices.size(); i++) + this.indices.add(src.indices.get(i).copy()); + } + + // include all fields here (standard and calculated) in the copy + if (src.getFields(true) == null) + return; + for (int i=0 ; i getFields(boolean includeCalculated) + { + if (includeCalculated) + return fields; + else + return getFields(); + } + + public List getFields() + { + if (standardFields == null) + standardFields = fields.stream().filter(f -> f.getValueExpression() == null).toList(); + return standardFields; + } + + public List getCalculatedFields() + { + if (calculatedFields == null) + calculatedFields = fields.stream().filter(f -> f.getValueExpression() != null).toList(); + return calculatedFields; + } + + public void setFields(List list) + { + fields = list; + + // reset the cached lists of fields so they will be recalculated on next call to getters + standardFields = null; + calculatedFields = null; + } + + public FieldType getFieldByName(String name) + { + for (FieldType field : getFields(true)) + { + if (field.getName() != null && field.getName().equalsIgnoreCase(name)) + return field; + } + return null; + } + + /** + * @return Indicates that the property can't be removed from the domain. The property may or may not be nullable. + */ + public boolean isMandatoryField(FieldType field) + { + if (mandatoryPropertyDescriptorNames == null || field.getName() == null) + { + return false; + } + return mandatoryPropertyDescriptorNames.contains(field.getName().toLowerCase()); + } + + public boolean isEditable(FieldType field) + { + return true; + } + + /** + * @return Indicates that the property is not allowed to be set as PHI + */ + public boolean allowsPhi(FieldType field) + { + return !(getPhiNotAllowedFieldNames() != null && field.getName() != null && getPhiNotAllowedFieldNames().contains(field.getName().toLowerCase())); + } + + /** + * @param mandatoryFieldNames names of property descriptors that must be present in this domain. Does not indicate that they must be non-nullable. + */ + public void setMandatoryFieldNames(Set mandatoryFieldNames) + { + this.mandatoryPropertyDescriptorNames = new HashSet<>(); + for (String mandatoryPropertyDescriptor : mandatoryFieldNames) + { + this.mandatoryPropertyDescriptorNames.add(mandatoryPropertyDescriptor.toLowerCase()); + } + } + + /** + * Get the list of property names that can't be removed from the domain. The set of mandatory fields is not modifiable in the designer. + */ + public Set getMandatoryFieldNames() + { + if (this.mandatoryPropertyDescriptorNames == null) + return Collections.emptySet(); + return Collections.unmodifiableSet(this.mandatoryPropertyDescriptorNames); + } + + public Set getReservedFieldNames() + { + return reservedFieldNames; + } + + /** + * @param reservedFieldNames can't create new fields with these names + */ + public void setReservedFieldNames(Set reservedFieldNames) + { + this.reservedFieldNames = new HashSet<>(); + for (String s : reservedFieldNames) + { + this.reservedFieldNames.add(s.toLowerCase()); + } + } + + public Set getReservedFieldNamePrefixes() + { + return this.reservedFieldNamePrefixes; + } + + public void setReservedFieldNamePrefixes(Set prefixes) + { + this.reservedFieldNamePrefixes = new HashSet<>(prefixes); + } + /** + * + * @param excludeFromExportFieldNames These fields will be suppressed from the export field list. Primary use case is to not export List key fields. + */ + public void setExcludeFromExportFieldNames(Set excludeFromExportFieldNames) + { + this.excludeFromExportFieldNames = new HashSet<>(); + for (String excludeFromExportFieldName : excludeFromExportFieldNames) + { + this.excludeFromExportFieldNames.add(excludeFromExportFieldName.toLowerCase()); + } + } + + public Set getExcludeFromExportFieldNames() + { + return excludeFromExportFieldNames; + } + + public boolean isExcludeFromExportField(FieldType field) + { + if (excludeFromExportFieldNames == null || field.getName() == null) + { + return false; + } + return excludeFromExportFieldNames.contains(field.getName().toLowerCase()); + } + + public Set getPhiNotAllowedFieldNames() + { + return phiNotAllowedFieldNames; + } + + public void setPhiNotAllowedFieldNames(Set phiNotAllowedFieldNames) + { + this.phiNotAllowedFieldNames = new HashSet<>(); + for (String fieldName : phiNotAllowedFieldNames) + { + this.phiNotAllowedFieldNames.add(fieldName.toLowerCase()); + } + } + + public DefaultValueType getDefaultDefaultValueType() + { + return defaultDefaultValueType; + } + + public DefaultValueType[] getDefaultValueOptions() + { + return defaultValueOptions; + } + + public void setDefaultValueOptions(DefaultValueType[] defaultOptions, DefaultValueType defaultDefault) + { + this.defaultDefaultValueType = defaultDefault; + this.defaultValueOptions = defaultOptions; + } + + public String getDefaultValuesURL() + { + if (defaultValuesURL == null) + return PropertyUtil.getRelativeURL("setDefaultValuesList", "list"); + return defaultValuesURL; + } + + public void setDefaultValuesURL(String defaultValuesURL) + { + this.defaultValuesURL = defaultValuesURL; + } +} diff --git a/api/src/org/labkey/api/data/ColumnRenderPropertiesImpl.java b/api/src/org/labkey/api/data/ColumnRenderPropertiesImpl.java index 06c4dcd35f8..1025106138f 100644 --- a/api/src/org/labkey/api/data/ColumnRenderPropertiesImpl.java +++ b/api/src/org/labkey/api/data/ColumnRenderPropertiesImpl.java @@ -48,6 +48,7 @@ public abstract class ColumnRenderPropertiesImpl implements MutableColumnRenderP public static final String STORAGE_UNIQUE_ID_CONCEPT_URI = "http://www.labkey.org/types#storageUniqueId"; public static final String STORAGE_UNIQUE_ID_SEQUENCE_PREFIX = "org.labkey.api.StorageUniqueId"; public static final String TEXT_CHOICE_CONCEPT_URI = "http://www.labkey.org/types#textChoice"; + //public static final String MULTI_VALUE_TEXT_CHOICE_CONCEPT_URI = "http://www.labkey.org/types#mvTextChoice"; public static final String NON_NEGATIVE_NUMBER_CONCEPT_URI = "http://www.labkey.org/types#nonNegativeNumber"; protected SortDirection _sortDirection = SortDirection.ASC; diff --git a/api/src/org/labkey/api/data/ConvertHelper.java b/api/src/org/labkey/api/data/ConvertHelper.java index a8918d2bc9a..9f55824144a 100644 --- a/api/src/org/labkey/api/data/ConvertHelper.java +++ b/api/src/org/labkey/api/data/ConvertHelper.java @@ -76,6 +76,7 @@ import java.io.File; import java.math.BigDecimal; import java.math.BigInteger; +import java.sql.Array; import java.sql.Blob; import java.sql.Clob; import java.sql.SQLException; @@ -189,6 +190,7 @@ protected void register() _register(new JSONTypeConverter(), JSONObject.class); _register(new ShortURLRecordConverter(), ShortURLRecord.class); _register(new ColumnHeaderType.Converter(), ColumnHeaderType.class); + _register(MultiChoice.Converter.getInstance(), MultiChoice.class); } diff --git a/api/src/org/labkey/api/data/JdbcType.java b/api/src/org/labkey/api/data/JdbcType.java index e427d25382a..480694853a6 100644 --- a/api/src/org/labkey/api/data/JdbcType.java +++ b/api/src/org/labkey/api/data/JdbcType.java @@ -21,6 +21,7 @@ import org.apache.commons.lang3.StringUtils; import org.jetbrains.annotations.NotNull; import org.jetbrains.annotations.Nullable; +import org.json.JSONArray; import org.junit.Assert; import org.junit.Test; import org.labkey.api.collections.IntHashMap; @@ -34,6 +35,7 @@ import java.sql.Time; import java.sql.Timestamp; import java.sql.Types; +import java.util.Arrays; import java.util.Collection; import java.util.Collections; import java.util.Date; @@ -300,7 +302,36 @@ protected Collection getSqlTypes() } }, - ARRAY(Types.ARRAY, Array.class), + ARRAY(Types.ARRAY, Array.class) + { + @Override + public Object convert(Object o) throws ConversionException + { + if ((o instanceof java.sql.Array array)) + return array; + if (o instanceof JSONArray jsonArray) + { + // convert jsonArray to array + Object[] elements = new Object[jsonArray.length()]; + for (int i = 0; i < jsonArray.length(); i++) + { + elements[i] = jsonArray.get(i); + } + return new MultiChoice.Array(Arrays.stream(elements)); + } + if (o instanceof Collection collection) + { + return new MultiChoice.Array(collection.stream().map(String::valueOf)); + } + if (o != null) + { + String s = String.valueOf(o); + return new MultiChoice.Array(Arrays.stream(new String[] {s})); + } + + return null; + } + }, NULL(Types.NULL, Object.class), diff --git a/api/src/org/labkey/api/data/MultiChoice.java b/api/src/org/labkey/api/data/MultiChoice.java index 7dda1715437..036a63ccf7a 100644 --- a/api/src/org/labkey/api/data/MultiChoice.java +++ b/api/src/org/labkey/api/data/MultiChoice.java @@ -21,6 +21,7 @@ import java.sql.ResultSet; import java.sql.SQLException; import java.sql.Types; +import java.text.Format; import java.util.ArrayList; import java.util.Arrays; import java.util.Collection; @@ -56,6 +57,11 @@ public DisplayColumn(ColumnInfo col) @Override public Object getValue(RenderContext ctx) + { + return getArrayValue(ctx); + } + + public Array getArrayValue(RenderContext ctx) { Object v = super.getValue(ctx); if (!(v instanceof java.sql.Array array)) @@ -129,6 +135,29 @@ public void renderInputCell(RenderContext ctx, HtmlWriter out) DIV(array.stream().map(v -> SPAN(at(style,"border:solid 1px black; border-radius:3px;"), v)) .collect(new JoinRenderable(HtmlString.SP)))); } + + @Override + public String getTsvFormattedValue(RenderContext ctx) + { + Array values = getArrayValue(ctx); + if (null != values && !values.isEmpty()) + { + return PageFlowUtil.joinValuesToStringForExport(values); + } + return null; + } + + @Override + public Object getExcelCompatibleValue(RenderContext ctx) + { + return getTsvFormattedValue(ctx); + } + + @Override + public Object getExportCompatibleValue(RenderContext ctx) + { + return getTsvFormattedValue(ctx); + } } diff --git a/api/src/org/labkey/api/data/PropertyStorageSpec.java b/api/src/org/labkey/api/data/PropertyStorageSpec.java index 9003a17fae0..db20b5052d3 100644 --- a/api/src/org/labkey/api/data/PropertyStorageSpec.java +++ b/api/src/org/labkey/api/data/PropertyStorageSpec.java @@ -169,6 +169,7 @@ public PropertyStorageSpec(PropertyDescriptor propertyDescriptor) setMvEnabled(propertyDescriptor.isMvEnabled()); setDescription(propertyDescriptor.getDescription()); setImportAliases(propertyDescriptor.getImportAliases()); + setTypeURI(propertyDescriptor.getRangeURI()); if (null != propertyDescriptor.getDatabaseDefaultValue()) setDefaultValue(propertyDescriptor.getDatabaseDefaultValue()); diff --git a/api/src/org/labkey/api/data/dialect/BasePostgreSqlDialect.java b/api/src/org/labkey/api/data/dialect/BasePostgreSqlDialect.java index f193447804b..c7e38673c4c 100644 --- a/api/src/org/labkey/api/data/dialect/BasePostgreSqlDialect.java +++ b/api/src/org/labkey/api/data/dialect/BasePostgreSqlDialect.java @@ -42,6 +42,7 @@ import org.labkey.api.data.TableInfo; import org.labkey.api.data.dialect.LimitRowsSqlGenerator.LimitRowsCustomizer; import org.labkey.api.data.dialect.LimitRowsSqlGenerator.StandardLimitRowsCustomizer; +import org.labkey.api.exp.PropertyType; import org.labkey.api.util.ExceptionUtil; import org.labkey.api.util.HtmlString; import org.labkey.api.util.StringUtilsLabKey; @@ -189,6 +190,8 @@ protected void addSqlTypeInts(Map sqlTypeIntMap) sqlTypeIntMap.put(Types.TIMESTAMP, "TIMESTAMP"); sqlTypeIntMap.put(Types.DOUBLE, "DOUBLE PRECISION"); sqlTypeIntMap.put(Types.FLOAT, "DOUBLE PRECISION"); + + sqlTypeIntMap.put(Types.ARRAY, "text[]"); // only support text arrays for now } @Override @@ -765,6 +768,10 @@ else if (prop.getJdbcType() == JdbcType.VARCHAR && (prop.getSize() == -1 || prop { return getSqlTypeName(JdbcType.LONGVARCHAR); } + else if (PropertyType.MULTI_CHOICE.getTypeUri().equals(prop.getTypeURI()) && prop.getJdbcType() == JdbcType.ARRAY) + { + return "text[]"; + } else { return getSqlTypeName(prop.getJdbcType()); diff --git a/api/src/org/labkey/api/exp/ObjectProperty.java b/api/src/org/labkey/api/exp/ObjectProperty.java index a0d4a0f1e2f..72a61647493 100644 --- a/api/src/org/labkey/api/exp/ObjectProperty.java +++ b/api/src/org/labkey/api/exp/ObjectProperty.java @@ -19,10 +19,12 @@ import org.labkey.api.data.BeanObjectFactory; import org.labkey.api.data.Container; import org.labkey.api.data.ContainerManager; +import org.labkey.api.data.MultiChoice; import org.labkey.api.data.MvUtil; import org.labkey.api.util.GUID; import java.io.File; +import java.sql.Array; import java.util.Collections; import java.util.Date; import java.util.Map; @@ -49,6 +51,7 @@ public class ObjectProperty extends OntologyManager.PropertyRow // ObjectProperty protected Identifiable objectValue; private Map _childProperties; + protected MultiChoice.Array arrayValue; // Don't delete this -- it's accessed via introspection public ObjectProperty() diff --git a/api/src/org/labkey/api/exp/PropertyType.java b/api/src/org/labkey/api/exp/PropertyType.java index a30ac08695f..d60ab77d3e2 100644 --- a/api/src/org/labkey/api/exp/PropertyType.java +++ b/api/src/org/labkey/api/exp/PropertyType.java @@ -1,1220 +1,1225 @@ -/* - * Copyright (c) 2008-2019 LabKey Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.labkey.api.exp; - -import org.apache.commons.beanutils.ConversionException; -import org.apache.commons.beanutils.ConvertUtils; -import org.apache.poi.ss.usermodel.Cell; -import org.apache.poi.ss.usermodel.CellType; -import org.fhcrc.cpas.exp.xml.SimpleTypeNames; -import org.jetbrains.annotations.NotNull; -import org.jetbrains.annotations.Nullable; -import org.labkey.api.attachments.AttachmentFile; -import org.labkey.api.collections.CaseInsensitiveHashMap; -import org.labkey.api.data.JdbcType; -import org.labkey.api.data.MultiChoice; -import org.labkey.api.data.NameGenerator; -import org.labkey.api.exp.OntologyManager.PropertyRow; -import org.labkey.api.reader.ExcelFactory; -import org.labkey.api.util.DateUtil; -import org.labkey.vfs.FileLike; - -import java.io.File; -import java.math.BigDecimal; -import java.nio.ByteBuffer; -import java.sql.Time; -import java.text.DateFormat; -import java.text.ParseException; -import java.text.SimpleDateFormat; -import java.util.Date; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.Objects; -import java.util.TimeZone; - -import static org.labkey.api.util.IntegerUtils.asIntegerElseNull; -import static org.labkey.api.util.IntegerUtils.asLongElseNull; - -/** - * TODO: Add more types? Entity, Lsid, User, ... - */ -public enum PropertyType -{ - BOOLEAN("http://www.w3.org/2001/XMLSchema#boolean", "Boolean", 'f', JdbcType.BOOLEAN, 10, null, CellType.BOOLEAN, Boolean.class, Boolean.TYPE) - { - @Override - protected Object convertExcelValue(Cell cell) throws ConversionException - { - return cell.getBooleanCellValue(); - } - - @Override - public Object convert(Object value) throws ConversionException - { - boolean boolValue = false; - if (value instanceof Boolean) - boolValue = (Boolean)value; - else if (null != value && !"".equals(value)) - boolValue = (Boolean) ConvertUtils.convert(value.toString(), Boolean.class); - return boolValue; - } - - @Override - public SimpleTypeNames.Enum getXmlBeanType() - { - return SimpleTypeNames.BOOLEAN; - } - - @Override - protected void init(PropertyRow row, Object value) - { - Boolean b = (Boolean)value; - row.floatValue = b == Boolean.TRUE ? 1.0 : 0.0; - } - - @Override - protected void setValue(ObjectProperty property, Object value) - { - Boolean boolValue = null; - if (value instanceof Boolean) - boolValue = (Boolean)value; - else if (null != value) - boolValue = (Boolean) ConvertUtils.convert(value.toString(), Boolean.class); - property.floatValue = boolValue == null ? null : boolValue == Boolean.TRUE ? 1.0 : 0.0; - } - - @Override - protected Object getValue(ObjectProperty property) - { - return property.floatValue == null ? null : property.floatValue.intValue() != 0 ? Boolean.TRUE : Boolean.FALSE; - } - - @Override - public Object getPreviewValue(@Nullable String prefix) - { - return Boolean.TRUE; - } - }, - STRING("http://www.w3.org/2001/XMLSchema#string", "String", 's', JdbcType.VARCHAR, 4000, "text", CellType.STRING, String.class) - { - @Override - protected Object convertExcelValue(Cell cell) throws ConversionException - { - return cell.getStringCellValue(); - } - - @Override - public Object convert(Object value) throws ConversionException - { - if (value instanceof String) - return value; - else - return ConvertUtils.convert(value); - } - - @Override - public SimpleTypeNames.Enum getXmlBeanType() - { - return SimpleTypeNames.STRING; - } - - @Override - protected void init(PropertyRow row, Object value) - { - row.stringValue = (String)value; - } - - @Override - protected void setValue(ObjectProperty property, Object value) - { - property.stringValue = value == null ? null : value.toString(); - } - - @Override - protected Object getValue(ObjectProperty property) - { - return property.getStringValue(); - } - - @Override - public Object getPreviewValue(@Nullable String prefix) - { - return prefix + "Value"; - } - }, - // NOT an XMLSchema type uri??? - MULTI_LINE("http://www.w3.org/2001/XMLSchema#multiLine", "MultiLine", 's', JdbcType.VARCHAR, 4000, "textarea", CellType.STRING, String.class) - { - @Override - protected Object convertExcelValue(Cell cell) throws ConversionException - { - return cell.getStringCellValue(); - } - - @Override - public Object convert(Object value) throws ConversionException - { - if (value instanceof String) - return value; - else - return ConvertUtils.convert(value); - } - - @Override - public SimpleTypeNames.Enum getXmlBeanType() - { - return SimpleTypeNames.STRING; - } - - @Override - protected void init(PropertyRow row, Object value) - { - row.stringValue = (String)value; - } - - @Override - protected void setValue(ObjectProperty property, Object value) - { - property.stringValue = value == null ? null : value.toString(); - } - - @Override - protected Object getValue(ObjectProperty property) - { - return property.getStringValue(); - } - - @Override - public Object getPreviewValue(@Nullable String prefix) - { - return prefix + "Value"; - } - }, - MULTI_CHOICE("http://cpas.fhcrc.org/exp/xml#multiChoice", "MultiChoice", '?' /* unsupported in exp.PropertyValues */, JdbcType.ARRAY, 0, "textarea", CellType.STRING, List.class) - { - @Override - protected Object convertExcelValue(Cell cell) throws ConversionException - { - return ConvertUtils.convert(cell.getStringCellValue(), MultiChoice.Array.class); - } - - @Override - public Object convert(Object value) throws ConversionException - { - return MultiChoice.Converter.getInstance().convert(MultiChoice.Array.class, value); - } - - @Override - public SimpleTypeNames.Enum getXmlBeanType() - { - return SimpleTypeNames.STRING; - } - - @Override - protected void init(PropertyRow row, Object value) - { - throw new UnsupportedOperationException("TODO MultiChoice"); - } - - @Override - protected void setValue(ObjectProperty property, Object value) - { - throw new UnsupportedOperationException("TODO MultiChoice"); - } - - @Override - protected Object getValue(ObjectProperty property) - { - throw new UnsupportedOperationException("TODO MultiChoice"); - } - - @Override - public Object getPreviewValue(@Nullable String prefix) - { - return "Option 1, Option 2"; - } - }, - RESOURCE("http://www.w3.org/2000/01/rdf-schema#Resource", "PropertyURI", 's', JdbcType.VARCHAR, 4000, null, CellType.STRING, Identifiable.class) - { - @Override - protected Object convertExcelValue(Cell cell) throws ConversionException - { - return cell.getStringCellValue(); - } - - @Override - public Object convert(Object value) throws ConversionException - { - if (null == value) - return null; - if (value instanceof Identifiable) - return ((Identifiable) value).getLSID(); - else - return value.toString(); - } - - @Override - public SimpleTypeNames.Enum getXmlBeanType() - { - return SimpleTypeNames.STRING; - } - - @Override - protected void init(PropertyRow row, Object value) - { - row.stringValue = (String)value; - } - - @Override - protected void setValue(ObjectProperty property, Object value) - { - if (value instanceof Identifiable) - { - property.stringValue = ((Identifiable) value).getLSID(); - property.objectValue = (Identifiable) value; - } - else if (null != value) - property.stringValue = value.toString(); - } - - @Override - protected Object getValue(ObjectProperty property) - { - if (null != property.objectValue) - return property.objectValue; - else - return property.getStringValue(); - } - - @Override - public Object getPreviewValue(@Nullable String prefix) - { - return prefix + "Value"; - } - }, - INTEGER("http://www.w3.org/2001/XMLSchema#int", "Integer", 'f', JdbcType.INTEGER, 10, null, CellType.NUMERIC, Integer.class, Integer.TYPE, Long.class, Long.TYPE) - { - @Override - protected Object convertExcelValue(Cell cell) throws ConversionException - { - return (int)cell.getNumericCellValue(); - } - - @Override - public Object convert(Object value) throws ConversionException - { - if (null == value) - return null; - if (asIntegerElseNull(value) instanceof Integer i) - return i; - else - return ConvertUtils.convert(value.toString(), Integer.class); - } - - @Override - public SimpleTypeNames.Enum getXmlBeanType() - { - return SimpleTypeNames.INTEGER; - } - - @Override - protected void init(PropertyRow row, Object value) - { - Number n = (Number) value; - if (null != n) - row.floatValue = n.doubleValue(); - } - - @Override - protected void setValue(ObjectProperty property, Object value) - { - if (null == value) - property.floatValue = null; - else if (asIntegerElseNull(value) instanceof Integer i) - property.floatValue = i.doubleValue(); - else - property.floatValue = (Double) ConvertUtils.convert(value.toString(), Double.class); - } - - @Override - protected Object getValue(ObjectProperty property) - { - return property.floatValue == null ? null : property.floatValue.intValue(); - } - - @Override - public Object getPreviewValue(@Nullable String prefix) - { - return Integer.valueOf(3); - } - }, - BIGINT("http://www.w3.org/2001/XMLSchema#long", "Long", 'f', JdbcType.BIGINT, 10, null, CellType.NUMERIC, Long.class, Long.TYPE) - { - @Override - protected Object convertExcelValue(Cell cell) throws ConversionException - { - return (int)cell.getNumericCellValue(); - } - - @Override - public Object convert(Object value) throws ConversionException - { - if (null == value) - return null; - if (asLongElseNull(value) instanceof Long l) - return l; - else - return ConvertUtils.convert(value.toString(), Long.class); - } - - @Override - public SimpleTypeNames.Enum getXmlBeanType() - { - throw new UnsupportedOperationException(); - } - - @Override - protected void init(PropertyRow row, Object value) - { - Number n = (Number) value; - if (null != n) - row.floatValue = n.doubleValue(); - } - - @Override - protected void setValue(ObjectProperty property, Object value) - { - if (null == value) - property.floatValue = null; - else if (asLongElseNull(value) instanceof Long l) - property.floatValue = l.doubleValue(); - else - property.floatValue = (Double) ConvertUtils.convert(value.toString(), Double.class); - } - - @Override - protected Object getValue(ObjectProperty property) - { - return property.floatValue == null ? null : property.floatValue.longValue(); - } - - @Override - public Object getPreviewValue(@Nullable String prefix) - { - return Integer.valueOf(3); - } - }, - // NOT an XMLSchema type uri??? - BINARY("http://www.w3.org/2001/XMLSchema#binary", "Binary", 'f', JdbcType.BINARY, 10, null, CellType.NUMERIC, ByteBuffer.class) - { - @Override - protected Object convertExcelValue(Cell cell) throws ConversionException - { - return (int)cell.getNumericCellValue(); - } - - @Override - public Object convert(Object value) throws ConversionException - { - if (null == value) - return null; - if (value instanceof ByteBuffer) - return value; - else - return ConvertUtils.convert(value.toString(), ByteBuffer.class); - } - - @Override - public SimpleTypeNames.Enum getXmlBeanType() - { - throw new UnsupportedOperationException(); - } - - @Override - protected void init(PropertyRow row, Object value) - { - throw new UnsupportedOperationException(); - } - - @Override - protected void setValue(ObjectProperty property, Object value) - { - if (null != value) - property.floatValue = (Double) ConvertUtils.convert(value.toString(), Double.class); - } - - @Override - protected Object getValue(ObjectProperty property) - { - throw new UnsupportedOperationException(); - } - }, - /** Stored as a path to a file on the server's file system */ - FILE_LINK("http://cpas.fhcrc.org/exp/xml#fileLink", "FileLink", 's', JdbcType.VARCHAR, 400, "file", CellType.STRING, File.class) - { - @Override - protected Object convertExcelValue(Cell cell) throws ConversionException - { - return cell.getStringCellValue(); - } - - @Override - public Object convert(Object value) throws ConversionException - { - if (null == value) - return null; - if (value instanceof File) - return ((File) value).getPath(); - else - return String.valueOf(value); - } - - @Override - public SimpleTypeNames.Enum getXmlBeanType() - { - return SimpleTypeNames.FILE_LINK; - } - - @Override - protected void init(PropertyRow row, Object value) - { - row.stringValue = (String)value; - } - - @Override - protected void setValue(ObjectProperty property, Object value) - { - if (value instanceof File f) - property.stringValue = f.getPath(); - else if (value instanceof FileLike fl) - property.stringValue = fl.toNioPathForRead().toString(); - else - property.stringValue = value == null ? null : value.toString(); - } - - @Override - protected Object getValue(ObjectProperty property) - { - String value = property.getStringValue(); - return value == null ? null : new File(value); - } - - @Override - public Object getPreviewValue(@Nullable String prefix) - { - return prefix + "Value"; - } - }, - /** Stored in the database as a BLOB using AttachmentService */ - ATTACHMENT("http://www.labkey.org/exp/xml#attachment", "Attachment", 's', JdbcType.VARCHAR, 100, "file", CellType.STRING, File.class) - { - @Override - protected Object convertExcelValue(Cell cell) throws ConversionException - { - return cell.getStringCellValue(); - } - - @Override - public Object convert(Object value) throws ConversionException - { - if (null == value) - return null; - if (value instanceof File) - return ((File) value).getPath(); - else - return String.valueOf(value); - } - - @Override - public SimpleTypeNames.Enum getXmlBeanType() - { - throw new UnsupportedOperationException(); - } - - @Override - protected void init(PropertyRow row, Object value) - { - row.stringValue = (String)value; - } - - @Override - protected void setValue(ObjectProperty property, Object value) - { - if (value instanceof AttachmentFile) - { - property.stringValue = ((AttachmentFile)value).getFilename(); - } - else - property.stringValue = value == null ? null : value.toString(); - } - - @Override - protected Object getValue(ObjectProperty property) - { - return property.getStringValue(); - } - - @Override - public Object getPreviewValue(@Nullable String prefix) - { - return prefix + "Value"; - } - }, - DATE_TIME("http://www.w3.org/2001/XMLSchema#dateTime", "DateTime", 'd', JdbcType.TIMESTAMP, 100, null, CellType.NUMERIC, Date.class) - { - @Override - protected Object convertExcelValue(Cell cell) throws ConversionException - { - Date date = cell.getDateCellValue(); - if (date != null) - { - DateFormat format = new SimpleDateFormat("MM/dd/yyyy GG HH:mm:ss.SSS"); - format.setTimeZone(TimeZone.getDefault()); - String s = format.format(date); - try - { - date = format.parse(s); - } - catch (ParseException e) - { - throw new ConversionException(e); - } -// int offset = TimeZone.getDefault().getOffset(date.getTime()); -// date.setTime(date.getTime() - offset); - } - return date; - } - - @Override - public Object convert(Object value) throws ConversionException - { - if (null == value) - return null; - if (value instanceof Date) - return value; - else - { - String strVal = value.toString(); - if (DateUtil.isSignedDuration(strVal)) - strVal = JdbcType.TIMESTAMP.convert(value).toString(); - return ConvertUtils.convert(strVal, Date.class); - } - } - - @Override - public SimpleTypeNames.Enum getXmlBeanType() - { - return SimpleTypeNames.DATE_TIME; - } - - @Override - protected void init(PropertyRow row, Object value) - { - row.dateTimeValue = new java.sql.Time(((java.util.Date)value).getTime()); - } - - @Override - protected void setValue(ObjectProperty property, Object value) - { - if (value instanceof Date) - property.dateTimeValue = (Date) value; - else if (null != value) - property.dateTimeValue = (Date) ConvertUtils.convert(value.toString(), Date.class); - } - - @Override - protected Object getValue(ObjectProperty property) - { - return property.dateTimeValue; - } - - - @Override - public Object getPreviewValue(@Nullable String prefix) - { - return NameGenerator.PREVIEW_DATETIME_VALUE; - } - }, - DATE("http://www.w3.org/2001/XMLSchema#date", "Date", 'd', JdbcType.DATE, 100, null, CellType.NUMERIC, Date.class) - { - @Override - protected Object convertExcelValue(Cell cell) throws ConversionException - { - return DateUtil.getDateOnly((Date)DATE_TIME.convertExcelValue(cell)); - } - - @Override - public Object convert(Object value) throws ConversionException - { - return DateUtil.getDateOnly((Date)DATE_TIME.convert(value)); - } - - @Override - public SimpleTypeNames.Enum getXmlBeanType() - { - return SimpleTypeNames.DATE_TIME; - } - - @Override - protected void init(PropertyRow row, Object value) - { - row.dateTimeValue = new java.sql.Date(((java.util.Date)value).getTime()); - } - - @Override - protected void setValue(ObjectProperty property, Object value) - { - if (value instanceof Date) - property.dateTimeValue = (Date) value; - else if (null != value) - property.dateTimeValue = (Date) ConvertUtils.convert(value.toString(), Date.class); - } - - @Override - protected Object getValue(ObjectProperty property) - { - return property.dateTimeValue; - } - - @Override - public Object getPreviewValue(@Nullable String prefix) - { - return NameGenerator.PREVIEW_DATE_VALUE; - } - }, - TIME("http://www.w3.org/2001/XMLSchema#time", "Time", 'd', JdbcType.TIME, 100, null, CellType.NUMERIC, java.sql.Time.class) - { - @Override - protected Object convertExcelValue(Cell cell) throws ConversionException - { - return DateUtil.getTimeOnly((Date)DATE_TIME.convertExcelValue(cell)); - } - - @Override - public Object convert(Object value) throws ConversionException - { - if (null == value) - return null; - - if (value instanceof Time) - return value; - - if (value instanceof Date) - return DateUtil.getTimeOnly((Date) value); - - try - { - return ConvertUtils.convert(value, Time.class); - } - catch (Exception ignore) - { - } - - return DateUtil.getTimeOnly((Date)DATE_TIME.convert(value)); - } - - @Override - public SimpleTypeNames.Enum getXmlBeanType() - { - return SimpleTypeNames.DATE_TIME; - } - - @Override - protected void init(PropertyRow row, Object value) - { - row.dateTimeValue = new java.sql.Time(((java.util.Date)value).getTime()); - } - - @Override - protected void setValue(ObjectProperty property, Object value) - { - if (value instanceof Time) - property.dateTimeValue = (Time) value; - else if (null != value) - property.dateTimeValue = (Time) ConvertUtils.convert(value.toString(), Time.class); - } - - @Override - protected Object getValue(ObjectProperty property) - { - return property.dateTimeValue; - } - - @Override - public Object getPreviewValue(@Nullable String prefix) - { - return NameGenerator.PREVIEW_TIME_VALUE; - } - }, - DOUBLE("http://www.w3.org/2001/XMLSchema#double", "Double", 'f', JdbcType.DOUBLE, 20, null, CellType.NUMERIC, Double.class, Double.TYPE, Float.class, Float.TYPE) - { - @Override - protected Object convertExcelValue(Cell cell) throws ConversionException - { - return cell.getNumericCellValue(); - } - - @Override - public Object convert(Object value) throws ConversionException - { - if (null == value) - return null; - if (value instanceof Double) - return value; - else - return ConvertUtils.convert(String.valueOf(value), Double.class); - } - - @Override - public SimpleTypeNames.Enum getXmlBeanType() - { - return SimpleTypeNames.DOUBLE; - } - - @Override - protected void init(PropertyRow row, Object value) - { - Number n = (Number) value; - if (null != n) - row.floatValue = n.doubleValue(); - } - - @Override - protected void setValue(ObjectProperty property, Object value) - { - if (value instanceof Double) - property.floatValue = (Double) value; - else if (null != value) - property.floatValue = (Double) ConvertUtils.convert(value.toString(), Double.class); - } - - @Override - protected Object getValue(ObjectProperty property) - { - return property.floatValue; - } - - @Override - public Object getPreviewValue(@Nullable String prefix) - { - return 12.34; - } - }, - FLOAT("http://www.w3.org/2001/XMLSchema#float", "Float", 'f', JdbcType.REAL, 20, null, CellType.NUMERIC, Float.class, Float.TYPE) - { - @Override - protected Object convertExcelValue(Cell cell) throws ConversionException - { - return cell.getNumericCellValue(); - } - - @Override - public Object convert(Object value) throws ConversionException - { - if (null == value) - return null; - if (value instanceof Float) - return value; - else - return ConvertUtils.convert(String.valueOf(value), Float.class); - } - - @Override - public SimpleTypeNames.Enum getXmlBeanType() - { - throw new UnsupportedOperationException(); - } - - @Override - protected void init(PropertyRow row, Object value) - { - Number n = (Number) value; - if (null != n) - row.floatValue = n.doubleValue(); - } - - @Override - protected void setValue(ObjectProperty property, Object value) - { - if (value instanceof Double) - property.floatValue = (Double) value; - else if (null != value) - property.floatValue = (Double) ConvertUtils.convert(value.toString(), Double.class); - } - - @Override - protected Object getValue(ObjectProperty property) - { - return property.floatValue; - } - - @Override - public Object getPreviewValue(@Nullable String prefix) - { - return 12.34; - } - }, - DECIMAL("http://www.w3.org/2001/XMLSchema#decimal", "Decimal", 'f', JdbcType.DECIMAL, 20, null, CellType.NUMERIC, BigDecimal.class) - { - @Override - protected Object convertExcelValue(Cell cell) throws ConversionException - { - return cell.getNumericCellValue(); - } - - @Override - public Object convert(Object value) throws ConversionException - { - if (null == value) - return null; - if (value instanceof BigDecimal) - return value; - else - return ConvertUtils.convert(String.valueOf(value), BigDecimal.class); - } - - @Override - public SimpleTypeNames.Enum getXmlBeanType() - { - throw new UnsupportedOperationException(); - } - - @Override - protected void init(PropertyRow row, Object value) - { - Number n = (Number) value; - if (null != n) - row.floatValue = n.doubleValue(); - } - - @Override - protected void setValue(ObjectProperty property, Object value) - { - if (null != value) - property.floatValue = (Double) ConvertUtils.convert(value.toString(), Double.class); - } - - @Override - protected Object getValue(ObjectProperty property) - { - return property.floatValue; - } - - @Override - public Object getPreviewValue(@Nullable String prefix) - { - return 12.34; - } - }, - XML_TEXT("http://cpas.fhcrc.org/exp/xml#text-xml", "XmlText", 's', JdbcType.LONGVARCHAR, 4000, null, CellType.STRING, null) - { - @Override - protected Object convertExcelValue(Cell cell) throws ConversionException - { - return cell.getStringCellValue(); - } - - @Override - public Object convert(Object value) throws ConversionException - { - if (value instanceof String) - return value; - else - return ConvertUtils.convert(value); - } - - @Override - public SimpleTypeNames.Enum getXmlBeanType() - { - throw new UnsupportedOperationException(); - } - - @Override - protected void init(PropertyRow row, Object value) - { - throw new UnsupportedOperationException(); - } - - @Override - protected void setValue(ObjectProperty property, Object value) - { - throw new UnsupportedOperationException(); - } - - @Override - protected Object getValue(ObjectProperty property) - { - return property.getStringValue(); - } - - @Override - public Object getPreviewValue(@Nullable String prefix) - { - return prefix + "Value"; - } - }; - - private final String typeURI; - private final String xarName; - private final char storageType; - private final CellType excelCellType; - private final @NotNull JdbcType jdbcType; - private final int scale; - private final String inputType; - private final Class javaType; - private final Class[] additionalTypes; - - private static Map uriToProperty; - private static Map xarToProperty = null; - - PropertyType(String typeURI, - String xarName, - char storageType, - @NotNull JdbcType jdbcType, - int scale, - String inputType, - CellType excelCellType, - Class javaType, - Class... additionalTypes) - { - this.typeURI = typeURI; - this.xarName = xarName; - this.storageType = storageType; - this.jdbcType = jdbcType; - this.scale = scale; - this.inputType = inputType; - this.javaType = javaType; - this.excelCellType = excelCellType; - this.additionalTypes = additionalTypes; - } - - public String getTypeUri() - { - return typeURI; - } - - public String getXmlName() - { - return xarName; - } - - public char getStorageType() - { - return storageType; - } - - @NotNull - public JdbcType getJdbcType() - { - return jdbcType; - } - - public int getScale() - { - return scale; - } - - @Nullable - public String getInputType() - { - return inputType; - } - - public Class getJavaType() - { - return javaType; - } - - public String getXarName() - { - return xarName; - } - - @NotNull - public static PropertyType getFromURI(String concept, String datatype) - { - return getFromURI(concept, datatype, RESOURCE); - } - - @Deprecated // Eliminate this along with PropertyRow? Or at least combine with setValue() below. - abstract protected void init(PropertyRow row, Object value); - abstract protected void setValue(ObjectProperty property, Object value); - abstract protected Object getValue(ObjectProperty property); - public Object getPreviewValue(@Nullable String prefix) - { - return getValue(null); - } - - static - { - Map m = new HashMap<>(); - - for (PropertyType t : values()) - { - String uri = t.getTypeUri(); - m.put(uri, t); - m.put(t.getXmlName(), t); - - if (uri.startsWith("http://www.w3.org/2001/XMLSchema#") || uri.startsWith("http://www.labkey.org/exp/xml#")) - { - String xsdName = uri.substring(uri.indexOf('#') + 1); - m.put("xsd:" + xsdName, t); - m.put(xsdName, t); - } - } - - uriToProperty = m; - } - - public static PropertyType getFromURI(@Nullable String concept, String datatype, PropertyType def) - { - PropertyType p = uriToProperty.get(concept); - - if (null == p) - { - p = uriToProperty.get(datatype); - if (null == p) - p = def; - } - - return p; - } - - @NotNull - public static PropertyType getFromXarName(String xarName) - { - return getFromXarName(xarName, RESOURCE); - } - - public static PropertyType getFromXarName(String xarName, PropertyType def) - { - if (null == xarToProperty) - { - Map m = new CaseInsensitiveHashMap<>(); - for (PropertyType t : values()) - { - m.put(t.getXmlName(), t); - } - xarToProperty = m; - } - - PropertyType p = xarToProperty.get(xarName); - - return null == p ? def : p; - } - - public static PropertyType getFromClass(Class clazz) - { - if (clazz == BigDecimal.class) - clazz = Double.class; - - for (PropertyType t : values()) - { - if (t.javaType == null) - continue; - if (t.javaType.isAssignableFrom(clazz)) - return t; - } - - // after trying the primary types, we then try any additional types: - for (PropertyType t : values()) - { - if (t.additionalTypes == null || t.additionalTypes.length == 0) - continue; - for (Class type : t.additionalTypes) - { - if (type.isAssignableFrom(clazz)) - return t; - } - } - return PropertyType.STRING; - } - - @NotNull - public static PropertyType getFromJdbcType(JdbcType jdbcType) - { - return Objects.requireNonNull(getFromJdbcType(jdbcType, true)); - } - - @Nullable - public static PropertyType getFromJdbcType(JdbcType jdbcType, boolean throwIfNotFound) - { - for (PropertyType t : values()) - { - if (t.jdbcType.equals(jdbcType)) - return t; - } - if (throwIfNotFound) - throw new IllegalArgumentException("No such JdbcType mapping: " + (null != jdbcType ? jdbcType.getClass().toString() : "null")); - else - return null; - } - - @Nullable - public static PropertyType getFromJdbcTypeName(String typeName) - { - for (PropertyType t : values()) - { - if (typeName.equalsIgnoreCase(t.jdbcType.name())) - return t; - } - return null; - } - - public abstract SimpleTypeNames.Enum getXmlBeanType(); - - protected abstract Object convertExcelValue(Cell cell) throws ConversionException; - - public abstract Object convert(Object value) throws ConversionException; - - public static Object getFromExcelCell(Cell cell) throws ConversionException - { - if (ExcelFactory.isCellNumeric(cell)) - { - // Ugly, the POI implementation doesn't expose an explicit date type - if (org.apache.poi.ss.usermodel.DateUtil.isCellDateFormatted(cell)) - return DATE_TIME.convertExcelValue(cell); - else - // special handling for the "number type": prefer double. - // Without this, we'd default to integer - return DOUBLE.convertExcelValue(cell); - } - - for (PropertyType t : values()) - { - if (t.excelCellType == cell.getCellType()) - return t.convertExcelValue(cell); - } - return ExcelFactory.getCellStringValue(cell); - } - - public String getValueTypeColumn() - { - switch (this.getStorageType()) - { - case 's': - return "stringValue"; - case 'd': - return "dateTimeValue"; - case 'f': - return "floatValue"; - default: - throw new IllegalArgumentException("Unknown property type: " + this); - } - } -} +/* + * Copyright (c) 2008-2019 LabKey Corporation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.labkey.api.exp; + +import org.apache.commons.beanutils.ConversionException; +import org.apache.commons.beanutils.ConvertUtils; +import org.apache.poi.ss.usermodel.Cell; +import org.apache.poi.ss.usermodel.CellType; +import org.fhcrc.cpas.exp.xml.SimpleTypeNames; +import org.jetbrains.annotations.NotNull; +import org.jetbrains.annotations.Nullable; +import org.labkey.api.attachments.AttachmentFile; +import org.labkey.api.collections.CaseInsensitiveHashMap; +import org.labkey.api.data.JdbcType; +import org.labkey.api.data.MultiChoice; +import org.labkey.api.data.NameGenerator; +import org.labkey.api.exp.OntologyManager.PropertyRow; +import org.labkey.api.reader.ExcelFactory; +import org.labkey.api.util.DateUtil; +import org.labkey.vfs.FileLike; + +import java.io.File; +import java.math.BigDecimal; +import java.nio.ByteBuffer; +import java.sql.Array; +import java.sql.SQLException; +import java.sql.Time; +import java.text.DateFormat; +import java.text.ParseException; +import java.text.SimpleDateFormat; +import java.util.Date; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.TimeZone; + +import static org.labkey.api.util.IntegerUtils.asIntegerElseNull; +import static org.labkey.api.util.IntegerUtils.asLongElseNull; + +/** + * TODO: Add more types? Entity, Lsid, User, ... + */ +public enum PropertyType +{ + BOOLEAN("http://www.w3.org/2001/XMLSchema#boolean", "Boolean", 'f', JdbcType.BOOLEAN, 10, null, CellType.BOOLEAN, Boolean.class, Boolean.TYPE) + { + @Override + protected Object convertExcelValue(Cell cell) throws ConversionException + { + return cell.getBooleanCellValue(); + } + + @Override + public Object convert(Object value) throws ConversionException + { + boolean boolValue = false; + if (value instanceof Boolean) + boolValue = (Boolean)value; + else if (null != value && !"".equals(value)) + boolValue = (Boolean) ConvertUtils.convert(value.toString(), Boolean.class); + return boolValue; + } + + @Override + public SimpleTypeNames.Enum getXmlBeanType() + { + return SimpleTypeNames.BOOLEAN; + } + + @Override + protected void init(PropertyRow row, Object value) + { + Boolean b = (Boolean)value; + row.floatValue = b == Boolean.TRUE ? 1.0 : 0.0; + } + + @Override + protected void setValue(ObjectProperty property, Object value) + { + Boolean boolValue = null; + if (value instanceof Boolean) + boolValue = (Boolean)value; + else if (null != value) + boolValue = (Boolean) ConvertUtils.convert(value.toString(), Boolean.class); + property.floatValue = boolValue == null ? null : boolValue == Boolean.TRUE ? 1.0 : 0.0; + } + + @Override + protected Object getValue(ObjectProperty property) + { + return property.floatValue == null ? null : property.floatValue.intValue() != 0 ? Boolean.TRUE : Boolean.FALSE; + } + + @Override + public Object getPreviewValue(@Nullable String prefix) + { + return Boolean.TRUE; + } + }, + STRING("http://www.w3.org/2001/XMLSchema#string", "String", 's', JdbcType.VARCHAR, 4000, "text", CellType.STRING, String.class) + { + @Override + protected Object convertExcelValue(Cell cell) throws ConversionException + { + return cell.getStringCellValue(); + } + + @Override + public Object convert(Object value) throws ConversionException + { + if (value instanceof String) + return value; + else + return ConvertUtils.convert(value); + } + + @Override + public SimpleTypeNames.Enum getXmlBeanType() + { + return SimpleTypeNames.STRING; + } + + @Override + protected void init(PropertyRow row, Object value) + { + row.stringValue = (String)value; + } + + @Override + protected void setValue(ObjectProperty property, Object value) + { + property.stringValue = value == null ? null : value.toString(); + } + + @Override + protected Object getValue(ObjectProperty property) + { + return property.getStringValue(); + } + + @Override + public Object getPreviewValue(@Nullable String prefix) + { + return prefix + "Value"; + } + }, + // NOT an XMLSchema type uri??? + MULTI_LINE("http://www.w3.org/2001/XMLSchema#multiLine", "MultiLine", 's', JdbcType.VARCHAR, 4000, "textarea", CellType.STRING, String.class) + { + @Override + protected Object convertExcelValue(Cell cell) throws ConversionException + { + return cell.getStringCellValue(); + } + + @Override + public Object convert(Object value) throws ConversionException + { + if (value instanceof String) + return value; + else + return ConvertUtils.convert(value); + } + + @Override + public SimpleTypeNames.Enum getXmlBeanType() + { + return SimpleTypeNames.STRING; + } + + @Override + protected void init(PropertyRow row, Object value) + { + row.stringValue = (String)value; + } + + @Override + protected void setValue(ObjectProperty property, Object value) + { + property.stringValue = value == null ? null : value.toString(); + } + + @Override + protected Object getValue(ObjectProperty property) + { + return property.getStringValue(); + } + + @Override + public Object getPreviewValue(@Nullable String prefix) + { + return prefix + "Value"; + } + }, + MULTI_CHOICE("http://cpas.fhcrc.org/exp/xml#multiChoice", "MultiChoice", '?' /* unsupported in exp.PropertyValues */, JdbcType.ARRAY, 0, "textarea", CellType.STRING, List.class) + { + @Override + protected Object convertExcelValue(Cell cell) throws ConversionException + { + return ConvertUtils.convert(cell.getStringCellValue(), MultiChoice.Array.class); + } + + @Override + public Object convert(Object value) throws ConversionException + { + return MultiChoice.Converter.getInstance().convert(MultiChoice.Array.class, value); + } + + @Override + public SimpleTypeNames.Enum getXmlBeanType() + { + return SimpleTypeNames.STRING; + } + + @Override + protected void init(PropertyRow row, Object value) + { + throw new UnsupportedOperationException("TODO MultiChoice"); + } + + @Override + protected void setValue(ObjectProperty property, Object value) + { + if ((value instanceof java.sql.Array array)) + property.arrayValue = MultiChoice.Array.from(array); + else if (value != null) + property.arrayValue = MultiChoice.Array.from(new Object[]{value}); + } + + @Override + protected Object getValue(ObjectProperty property) + { + return property.arrayValue; + } + + @Override + public Object getPreviewValue(@Nullable String prefix) + { + return "Option 1, Option 2"; + } + }, + RESOURCE("http://www.w3.org/2000/01/rdf-schema#Resource", "PropertyURI", 's', JdbcType.VARCHAR, 4000, null, CellType.STRING, Identifiable.class) + { + @Override + protected Object convertExcelValue(Cell cell) throws ConversionException + { + return cell.getStringCellValue(); + } + + @Override + public Object convert(Object value) throws ConversionException + { + if (null == value) + return null; + if (value instanceof Identifiable) + return ((Identifiable) value).getLSID(); + else + return value.toString(); + } + + @Override + public SimpleTypeNames.Enum getXmlBeanType() + { + return SimpleTypeNames.STRING; + } + + @Override + protected void init(PropertyRow row, Object value) + { + row.stringValue = (String)value; + } + + @Override + protected void setValue(ObjectProperty property, Object value) + { + if (value instanceof Identifiable) + { + property.stringValue = ((Identifiable) value).getLSID(); + property.objectValue = (Identifiable) value; + } + else if (null != value) + property.stringValue = value.toString(); + } + + @Override + protected Object getValue(ObjectProperty property) + { + if (null != property.objectValue) + return property.objectValue; + else + return property.getStringValue(); + } + + @Override + public Object getPreviewValue(@Nullable String prefix) + { + return prefix + "Value"; + } + }, + INTEGER("http://www.w3.org/2001/XMLSchema#int", "Integer", 'f', JdbcType.INTEGER, 10, null, CellType.NUMERIC, Integer.class, Integer.TYPE, Long.class, Long.TYPE) + { + @Override + protected Object convertExcelValue(Cell cell) throws ConversionException + { + return (int)cell.getNumericCellValue(); + } + + @Override + public Object convert(Object value) throws ConversionException + { + if (null == value) + return null; + if (asIntegerElseNull(value) instanceof Integer i) + return i; + else + return ConvertUtils.convert(value.toString(), Integer.class); + } + + @Override + public SimpleTypeNames.Enum getXmlBeanType() + { + return SimpleTypeNames.INTEGER; + } + + @Override + protected void init(PropertyRow row, Object value) + { + Number n = (Number) value; + if (null != n) + row.floatValue = n.doubleValue(); + } + + @Override + protected void setValue(ObjectProperty property, Object value) + { + if (null == value) + property.floatValue = null; + else if (asIntegerElseNull(value) instanceof Integer i) + property.floatValue = i.doubleValue(); + else + property.floatValue = (Double) ConvertUtils.convert(value.toString(), Double.class); + } + + @Override + protected Object getValue(ObjectProperty property) + { + return property.floatValue == null ? null : property.floatValue.intValue(); + } + + @Override + public Object getPreviewValue(@Nullable String prefix) + { + return Integer.valueOf(3); + } + }, + BIGINT("http://www.w3.org/2001/XMLSchema#long", "Long", 'f', JdbcType.BIGINT, 10, null, CellType.NUMERIC, Long.class, Long.TYPE) + { + @Override + protected Object convertExcelValue(Cell cell) throws ConversionException + { + return (int)cell.getNumericCellValue(); + } + + @Override + public Object convert(Object value) throws ConversionException + { + if (null == value) + return null; + if (asLongElseNull(value) instanceof Long l) + return l; + else + return ConvertUtils.convert(value.toString(), Long.class); + } + + @Override + public SimpleTypeNames.Enum getXmlBeanType() + { + throw new UnsupportedOperationException(); + } + + @Override + protected void init(PropertyRow row, Object value) + { + Number n = (Number) value; + if (null != n) + row.floatValue = n.doubleValue(); + } + + @Override + protected void setValue(ObjectProperty property, Object value) + { + if (null == value) + property.floatValue = null; + else if (asLongElseNull(value) instanceof Long l) + property.floatValue = l.doubleValue(); + else + property.floatValue = (Double) ConvertUtils.convert(value.toString(), Double.class); + } + + @Override + protected Object getValue(ObjectProperty property) + { + return property.floatValue == null ? null : property.floatValue.longValue(); + } + + @Override + public Object getPreviewValue(@Nullable String prefix) + { + return Integer.valueOf(3); + } + }, + // NOT an XMLSchema type uri??? + BINARY("http://www.w3.org/2001/XMLSchema#binary", "Binary", 'f', JdbcType.BINARY, 10, null, CellType.NUMERIC, ByteBuffer.class) + { + @Override + protected Object convertExcelValue(Cell cell) throws ConversionException + { + return (int)cell.getNumericCellValue(); + } + + @Override + public Object convert(Object value) throws ConversionException + { + if (null == value) + return null; + if (value instanceof ByteBuffer) + return value; + else + return ConvertUtils.convert(value.toString(), ByteBuffer.class); + } + + @Override + public SimpleTypeNames.Enum getXmlBeanType() + { + throw new UnsupportedOperationException(); + } + + @Override + protected void init(PropertyRow row, Object value) + { + throw new UnsupportedOperationException(); + } + + @Override + protected void setValue(ObjectProperty property, Object value) + { + if (null != value) + property.floatValue = (Double) ConvertUtils.convert(value.toString(), Double.class); + } + + @Override + protected Object getValue(ObjectProperty property) + { + throw new UnsupportedOperationException(); + } + }, + /** Stored as a path to a file on the server's file system */ + FILE_LINK("http://cpas.fhcrc.org/exp/xml#fileLink", "FileLink", 's', JdbcType.VARCHAR, 400, "file", CellType.STRING, File.class) + { + @Override + protected Object convertExcelValue(Cell cell) throws ConversionException + { + return cell.getStringCellValue(); + } + + @Override + public Object convert(Object value) throws ConversionException + { + if (null == value) + return null; + if (value instanceof File) + return ((File) value).getPath(); + else + return String.valueOf(value); + } + + @Override + public SimpleTypeNames.Enum getXmlBeanType() + { + return SimpleTypeNames.FILE_LINK; + } + + @Override + protected void init(PropertyRow row, Object value) + { + row.stringValue = (String)value; + } + + @Override + protected void setValue(ObjectProperty property, Object value) + { + if (value instanceof File f) + property.stringValue = f.getPath(); + else if (value instanceof FileLike fl) + property.stringValue = fl.toNioPathForRead().toString(); + else + property.stringValue = value == null ? null : value.toString(); + } + + @Override + protected Object getValue(ObjectProperty property) + { + String value = property.getStringValue(); + return value == null ? null : new File(value); + } + + @Override + public Object getPreviewValue(@Nullable String prefix) + { + return prefix + "Value"; + } + }, + /** Stored in the database as a BLOB using AttachmentService */ + ATTACHMENT("http://www.labkey.org/exp/xml#attachment", "Attachment", 's', JdbcType.VARCHAR, 100, "file", CellType.STRING, File.class) + { + @Override + protected Object convertExcelValue(Cell cell) throws ConversionException + { + return cell.getStringCellValue(); + } + + @Override + public Object convert(Object value) throws ConversionException + { + if (null == value) + return null; + if (value instanceof File) + return ((File) value).getPath(); + else + return String.valueOf(value); + } + + @Override + public SimpleTypeNames.Enum getXmlBeanType() + { + throw new UnsupportedOperationException(); + } + + @Override + protected void init(PropertyRow row, Object value) + { + row.stringValue = (String)value; + } + + @Override + protected void setValue(ObjectProperty property, Object value) + { + if (value instanceof AttachmentFile) + { + property.stringValue = ((AttachmentFile)value).getFilename(); + } + else + property.stringValue = value == null ? null : value.toString(); + } + + @Override + protected Object getValue(ObjectProperty property) + { + return property.getStringValue(); + } + + @Override + public Object getPreviewValue(@Nullable String prefix) + { + return prefix + "Value"; + } + }, + DATE_TIME("http://www.w3.org/2001/XMLSchema#dateTime", "DateTime", 'd', JdbcType.TIMESTAMP, 100, null, CellType.NUMERIC, Date.class) + { + @Override + protected Object convertExcelValue(Cell cell) throws ConversionException + { + Date date = cell.getDateCellValue(); + if (date != null) + { + DateFormat format = new SimpleDateFormat("MM/dd/yyyy GG HH:mm:ss.SSS"); + format.setTimeZone(TimeZone.getDefault()); + String s = format.format(date); + try + { + date = format.parse(s); + } + catch (ParseException e) + { + throw new ConversionException(e); + } +// int offset = TimeZone.getDefault().getOffset(date.getTime()); +// date.setTime(date.getTime() - offset); + } + return date; + } + + @Override + public Object convert(Object value) throws ConversionException + { + if (null == value) + return null; + if (value instanceof Date) + return value; + else + { + String strVal = value.toString(); + if (DateUtil.isSignedDuration(strVal)) + strVal = JdbcType.TIMESTAMP.convert(value).toString(); + return ConvertUtils.convert(strVal, Date.class); + } + } + + @Override + public SimpleTypeNames.Enum getXmlBeanType() + { + return SimpleTypeNames.DATE_TIME; + } + + @Override + protected void init(PropertyRow row, Object value) + { + row.dateTimeValue = new java.sql.Time(((java.util.Date)value).getTime()); + } + + @Override + protected void setValue(ObjectProperty property, Object value) + { + if (value instanceof Date) + property.dateTimeValue = (Date) value; + else if (null != value) + property.dateTimeValue = (Date) ConvertUtils.convert(value.toString(), Date.class); + } + + @Override + protected Object getValue(ObjectProperty property) + { + return property.dateTimeValue; + } + + + @Override + public Object getPreviewValue(@Nullable String prefix) + { + return NameGenerator.PREVIEW_DATETIME_VALUE; + } + }, + DATE("http://www.w3.org/2001/XMLSchema#date", "Date", 'd', JdbcType.DATE, 100, null, CellType.NUMERIC, Date.class) + { + @Override + protected Object convertExcelValue(Cell cell) throws ConversionException + { + return DateUtil.getDateOnly((Date)DATE_TIME.convertExcelValue(cell)); + } + + @Override + public Object convert(Object value) throws ConversionException + { + return DateUtil.getDateOnly((Date)DATE_TIME.convert(value)); + } + + @Override + public SimpleTypeNames.Enum getXmlBeanType() + { + return SimpleTypeNames.DATE_TIME; + } + + @Override + protected void init(PropertyRow row, Object value) + { + row.dateTimeValue = new java.sql.Date(((java.util.Date)value).getTime()); + } + + @Override + protected void setValue(ObjectProperty property, Object value) + { + if (value instanceof Date) + property.dateTimeValue = (Date) value; + else if (null != value) + property.dateTimeValue = (Date) ConvertUtils.convert(value.toString(), Date.class); + } + + @Override + protected Object getValue(ObjectProperty property) + { + return property.dateTimeValue; + } + + @Override + public Object getPreviewValue(@Nullable String prefix) + { + return NameGenerator.PREVIEW_DATE_VALUE; + } + }, + TIME("http://www.w3.org/2001/XMLSchema#time", "Time", 'd', JdbcType.TIME, 100, null, CellType.NUMERIC, java.sql.Time.class) + { + @Override + protected Object convertExcelValue(Cell cell) throws ConversionException + { + return DateUtil.getTimeOnly((Date)DATE_TIME.convertExcelValue(cell)); + } + + @Override + public Object convert(Object value) throws ConversionException + { + if (null == value) + return null; + + if (value instanceof Time) + return value; + + if (value instanceof Date) + return DateUtil.getTimeOnly((Date) value); + + try + { + return ConvertUtils.convert(value, Time.class); + } + catch (Exception ignore) + { + } + + return DateUtil.getTimeOnly((Date)DATE_TIME.convert(value)); + } + + @Override + public SimpleTypeNames.Enum getXmlBeanType() + { + return SimpleTypeNames.DATE_TIME; + } + + @Override + protected void init(PropertyRow row, Object value) + { + row.dateTimeValue = new java.sql.Time(((java.util.Date)value).getTime()); + } + + @Override + protected void setValue(ObjectProperty property, Object value) + { + if (value instanceof Time) + property.dateTimeValue = (Time) value; + else if (null != value) + property.dateTimeValue = (Time) ConvertUtils.convert(value.toString(), Time.class); + } + + @Override + protected Object getValue(ObjectProperty property) + { + return property.dateTimeValue; + } + + @Override + public Object getPreviewValue(@Nullable String prefix) + { + return NameGenerator.PREVIEW_TIME_VALUE; + } + }, + DOUBLE("http://www.w3.org/2001/XMLSchema#double", "Double", 'f', JdbcType.DOUBLE, 20, null, CellType.NUMERIC, Double.class, Double.TYPE, Float.class, Float.TYPE) + { + @Override + protected Object convertExcelValue(Cell cell) throws ConversionException + { + return cell.getNumericCellValue(); + } + + @Override + public Object convert(Object value) throws ConversionException + { + if (null == value) + return null; + if (value instanceof Double) + return value; + else + return ConvertUtils.convert(String.valueOf(value), Double.class); + } + + @Override + public SimpleTypeNames.Enum getXmlBeanType() + { + return SimpleTypeNames.DOUBLE; + } + + @Override + protected void init(PropertyRow row, Object value) + { + Number n = (Number) value; + if (null != n) + row.floatValue = n.doubleValue(); + } + + @Override + protected void setValue(ObjectProperty property, Object value) + { + if (value instanceof Double) + property.floatValue = (Double) value; + else if (null != value) + property.floatValue = (Double) ConvertUtils.convert(value.toString(), Double.class); + } + + @Override + protected Object getValue(ObjectProperty property) + { + return property.floatValue; + } + + @Override + public Object getPreviewValue(@Nullable String prefix) + { + return 12.34; + } + }, + FLOAT("http://www.w3.org/2001/XMLSchema#float", "Float", 'f', JdbcType.REAL, 20, null, CellType.NUMERIC, Float.class, Float.TYPE) + { + @Override + protected Object convertExcelValue(Cell cell) throws ConversionException + { + return cell.getNumericCellValue(); + } + + @Override + public Object convert(Object value) throws ConversionException + { + if (null == value) + return null; + if (value instanceof Float) + return value; + else + return ConvertUtils.convert(String.valueOf(value), Float.class); + } + + @Override + public SimpleTypeNames.Enum getXmlBeanType() + { + throw new UnsupportedOperationException(); + } + + @Override + protected void init(PropertyRow row, Object value) + { + Number n = (Number) value; + if (null != n) + row.floatValue = n.doubleValue(); + } + + @Override + protected void setValue(ObjectProperty property, Object value) + { + if (value instanceof Double) + property.floatValue = (Double) value; + else if (null != value) + property.floatValue = (Double) ConvertUtils.convert(value.toString(), Double.class); + } + + @Override + protected Object getValue(ObjectProperty property) + { + return property.floatValue; + } + + @Override + public Object getPreviewValue(@Nullable String prefix) + { + return 12.34; + } + }, + DECIMAL("http://www.w3.org/2001/XMLSchema#decimal", "Decimal", 'f', JdbcType.DECIMAL, 20, null, CellType.NUMERIC, BigDecimal.class) + { + @Override + protected Object convertExcelValue(Cell cell) throws ConversionException + { + return cell.getNumericCellValue(); + } + + @Override + public Object convert(Object value) throws ConversionException + { + if (null == value) + return null; + if (value instanceof BigDecimal) + return value; + else + return ConvertUtils.convert(String.valueOf(value), BigDecimal.class); + } + + @Override + public SimpleTypeNames.Enum getXmlBeanType() + { + throw new UnsupportedOperationException(); + } + + @Override + protected void init(PropertyRow row, Object value) + { + Number n = (Number) value; + if (null != n) + row.floatValue = n.doubleValue(); + } + + @Override + protected void setValue(ObjectProperty property, Object value) + { + if (null != value) + property.floatValue = (Double) ConvertUtils.convert(value.toString(), Double.class); + } + + @Override + protected Object getValue(ObjectProperty property) + { + return property.floatValue; + } + + @Override + public Object getPreviewValue(@Nullable String prefix) + { + return 12.34; + } + }, + XML_TEXT("http://cpas.fhcrc.org/exp/xml#text-xml", "XmlText", 's', JdbcType.LONGVARCHAR, 4000, null, CellType.STRING, null) + { + @Override + protected Object convertExcelValue(Cell cell) throws ConversionException + { + return cell.getStringCellValue(); + } + + @Override + public Object convert(Object value) throws ConversionException + { + if (value instanceof String) + return value; + else + return ConvertUtils.convert(value); + } + + @Override + public SimpleTypeNames.Enum getXmlBeanType() + { + throw new UnsupportedOperationException(); + } + + @Override + protected void init(PropertyRow row, Object value) + { + throw new UnsupportedOperationException(); + } + + @Override + protected void setValue(ObjectProperty property, Object value) + { + throw new UnsupportedOperationException(); + } + + @Override + protected Object getValue(ObjectProperty property) + { + return property.getStringValue(); + } + + @Override + public Object getPreviewValue(@Nullable String prefix) + { + return prefix + "Value"; + } + }; + + private final String typeURI; + private final String xarName; + private final char storageType; + private final CellType excelCellType; + private final @NotNull JdbcType jdbcType; + private final int scale; + private final String inputType; + private final Class javaType; + private final Class[] additionalTypes; + + private static Map uriToProperty; + private static Map xarToProperty = null; + + PropertyType(String typeURI, + String xarName, + char storageType, + @NotNull JdbcType jdbcType, + int scale, + String inputType, + CellType excelCellType, + Class javaType, + Class... additionalTypes) + { + this.typeURI = typeURI; + this.xarName = xarName; + this.storageType = storageType; + this.jdbcType = jdbcType; + this.scale = scale; + this.inputType = inputType; + this.javaType = javaType; + this.excelCellType = excelCellType; + this.additionalTypes = additionalTypes; + } + + public String getTypeUri() + { + return typeURI; + } + + public String getXmlName() + { + return xarName; + } + + public char getStorageType() + { + return storageType; + } + + @NotNull + public JdbcType getJdbcType() + { + return jdbcType; + } + + public int getScale() + { + return scale; + } + + @Nullable + public String getInputType() + { + return inputType; + } + + public Class getJavaType() + { + return javaType; + } + + public String getXarName() + { + return xarName; + } + + @NotNull + public static PropertyType getFromURI(String concept, String datatype) + { + return getFromURI(concept, datatype, RESOURCE); + } + + @Deprecated // Eliminate this along with PropertyRow? Or at least combine with setValue() below. + abstract protected void init(PropertyRow row, Object value); + abstract protected void setValue(ObjectProperty property, Object value); + abstract protected Object getValue(ObjectProperty property); + public Object getPreviewValue(@Nullable String prefix) + { + return getValue(null); + } + + static + { + Map m = new HashMap<>(); + + for (PropertyType t : values()) + { + String uri = t.getTypeUri(); + m.put(uri, t); + m.put(t.getXmlName(), t); + + if (uri.startsWith("http://www.w3.org/2001/XMLSchema#") || uri.startsWith("http://www.labkey.org/exp/xml#")) + { + String xsdName = uri.substring(uri.indexOf('#') + 1); + m.put("xsd:" + xsdName, t); + m.put(xsdName, t); + } + } + + uriToProperty = m; + } + + public static PropertyType getFromURI(@Nullable String concept, String datatype, PropertyType def) + { + PropertyType p = uriToProperty.get(concept); + + if (null == p) + { + p = uriToProperty.get(datatype); + if (null == p) + p = def; + } + + return p; + } + + @NotNull + public static PropertyType getFromXarName(String xarName) + { + return getFromXarName(xarName, RESOURCE); + } + + public static PropertyType getFromXarName(String xarName, PropertyType def) + { + if (null == xarToProperty) + { + Map m = new CaseInsensitiveHashMap<>(); + for (PropertyType t : values()) + { + m.put(t.getXmlName(), t); + } + xarToProperty = m; + } + + PropertyType p = xarToProperty.get(xarName); + + return null == p ? def : p; + } + + public static PropertyType getFromClass(Class clazz) + { + if (clazz == BigDecimal.class) + clazz = Double.class; + + for (PropertyType t : values()) + { + if (t.javaType == null) + continue; + if (t.javaType.isAssignableFrom(clazz)) + return t; + } + + // after trying the primary types, we then try any additional types: + for (PropertyType t : values()) + { + if (t.additionalTypes == null || t.additionalTypes.length == 0) + continue; + for (Class type : t.additionalTypes) + { + if (type.isAssignableFrom(clazz)) + return t; + } + } + return PropertyType.STRING; + } + + @NotNull + public static PropertyType getFromJdbcType(JdbcType jdbcType) + { + return Objects.requireNonNull(getFromJdbcType(jdbcType, true)); + } + + @Nullable + public static PropertyType getFromJdbcType(JdbcType jdbcType, boolean throwIfNotFound) + { + for (PropertyType t : values()) + { + if (t.jdbcType.equals(jdbcType)) + return t; + } + if (throwIfNotFound) + throw new IllegalArgumentException("No such JdbcType mapping: " + (null != jdbcType ? jdbcType.getClass().toString() : "null")); + else + return null; + } + + @Nullable + public static PropertyType getFromJdbcTypeName(String typeName) + { + for (PropertyType t : values()) + { + if (typeName.equalsIgnoreCase(t.jdbcType.name())) + return t; + } + return null; + } + + public abstract SimpleTypeNames.Enum getXmlBeanType(); + + protected abstract Object convertExcelValue(Cell cell) throws ConversionException; + + public abstract Object convert(Object value) throws ConversionException; + + public static Object getFromExcelCell(Cell cell) throws ConversionException + { + if (ExcelFactory.isCellNumeric(cell)) + { + // Ugly, the POI implementation doesn't expose an explicit date type + if (org.apache.poi.ss.usermodel.DateUtil.isCellDateFormatted(cell)) + return DATE_TIME.convertExcelValue(cell); + else + // special handling for the "number type": prefer double. + // Without this, we'd default to integer + return DOUBLE.convertExcelValue(cell); + } + + for (PropertyType t : values()) + { + if (t.excelCellType == cell.getCellType()) + return t.convertExcelValue(cell); + } + return ExcelFactory.getCellStringValue(cell); + } + + public String getValueTypeColumn() + { + switch (this.getStorageType()) + { + case 's': + return "stringValue"; + case 'd': + return "dateTimeValue"; + case 'f': + return "floatValue"; + default: + throw new IllegalArgumentException("Unknown property type: " + this); + } + } +} diff --git a/api/src/org/labkey/api/exp/api/SampleTypeDomainKind.java b/api/src/org/labkey/api/exp/api/SampleTypeDomainKind.java index da47258672a..6b06f23ebb0 100644 --- a/api/src/org/labkey/api/exp/api/SampleTypeDomainKind.java +++ b/api/src/org/labkey/api/exp/api/SampleTypeDomainKind.java @@ -206,6 +206,12 @@ public boolean allowFileLinkProperties() return true; } + @Override + public boolean allowMultiChoiceProperties() + { + return true; + } + @Override public boolean allowTimepointProperties() { diff --git a/api/src/org/labkey/api/exp/property/DomainKind.java b/api/src/org/labkey/api/exp/property/DomainKind.java index f2a8ff5aad9..65bb8a58ff2 100644 --- a/api/src/org/labkey/api/exp/property/DomainKind.java +++ b/api/src/org/labkey/api/exp/property/DomainKind.java @@ -1,439 +1,439 @@ -/* - * Copyright (c) 2008-2019 LabKey Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.labkey.api.exp.property; - -import org.apache.logging.log4j.Logger; -import org.jetbrains.annotations.NotNull; -import org.jetbrains.annotations.Nullable; -import org.labkey.api.collections.CaseInsensitiveHashSet; -import org.labkey.api.data.Container; -import org.labkey.api.data.ContainerFilter; -import org.labkey.api.data.DbSchemaType; -import org.labkey.api.data.DbScope; -import org.labkey.api.data.NameExpressionValidationResult; -import org.labkey.api.data.PropertyStorageSpec; -import org.labkey.api.data.SQLFragment; -import org.labkey.api.data.SchemaTableInfo; -import org.labkey.api.data.TableInfo; -import org.labkey.api.data.UpdateableTableInfo; -import org.labkey.api.exp.Handler; -import org.labkey.api.exp.PropertyDescriptor; -import org.labkey.api.exp.TemplateInfo; -import org.labkey.api.gwt.client.DefaultValueType; -import org.labkey.api.gwt.client.model.GWTDomain; -import org.labkey.api.gwt.client.model.GWTPropertyDescriptor; -import org.labkey.api.query.UserSchema; -import org.labkey.api.query.ValidationException; -import org.labkey.api.security.User; -import org.labkey.api.security.UserPrincipal; -import org.labkey.api.security.permissions.Permission; -import org.labkey.api.util.logging.LogHelper; -import org.labkey.api.view.ActionURL; -import org.labkey.api.view.NavTree; -import org.labkey.api.view.UnauthorizedException; -import org.labkey.api.writer.ContainerUser; -import org.labkey.data.xml.domainTemplate.DomainTemplateType; - -import java.util.Collections; -import java.util.LinkedHashSet; -import java.util.List; -import java.util.Map; -import java.util.Set; -import java.util.stream.Collectors; - -abstract public class DomainKind implements Handler -{ - public static final Logger LOG = LogHelper.getLogger(DomainKind.class, "Generic domain kind activities."); - abstract public String getKindName(); - - /** - * Return a class of DomainKind's bean which carries domain specific properties. - * This class will be used when marshalling/unmarshalling via Jackson during Create and Save/Update Domain - * @return Class of DomainKind's bean with domain specific properties - */ - abstract public Class getTypeClass(); - - /** - * Give the domain kind a chance to process / map the arguments before converting it to the DomainKind properties object. - * @param arguments initial arguments coming from the caller - * @return updated arguments - */ - public Map processArguments(Container container, User user, Map arguments) - { - return arguments; - } - - abstract public String getTypeLabel(Domain domain); - abstract public SQLFragment sqlObjectIdsInDomain(Domain domain); - - /** - * Create a DomainURI for a Domain that may or may not exist yet. - */ - abstract public String generateDomainURI(String schemaName, String queryName, Container container, User user); - - abstract public @Nullable ActionURL urlShowData(Domain domain, ContainerUser containerUser); - abstract public @Nullable ActionURL urlEditDefinition(Domain domain, ContainerUser containerUser); - abstract public ActionURL urlCreateDefinition(String schemaName, String queryName, Container container, User user); - - // Override to return a non-null String and the generic domain editor will display it in an "Instructions" webpart above the field properties - public @Nullable String getDomainEditorInstructions() - { - return null; - } - - abstract public boolean canCreateDefinition(User user, Container container); - - abstract public boolean canEditDefinition(User user, Domain domain); - - abstract public boolean canDeleteDefinition(User user, Domain domain); - - // Override to customize the nav trail on shared pages like edit domain - abstract public void addNavTrail(NavTree root, Container c, User user); - - // Do any special handling before a PropertyDescriptor is deleted -- do nothing by default - abstract public void deletePropertyDescriptor(Domain domain, User user, PropertyDescriptor pd); - - /** - * Return the set of names that should not be allowed for properties. E.g. - * the names of columns from the hard table underlying this type - * @return set of strings containing the names. This will be compared ignoring case - */ - abstract public Set getReservedPropertyNames(Domain domain, User user); - - public Set getReservedPropertyNames(Domain domain, User user, boolean forCreate) - { - return getReservedPropertyNames(domain, user); - } - - public Set getReservedPropertyNamePrefixes() - { - return Collections.emptySet(); - } - - /** - * Return the set of names that are always required and cannot subsequently - * be deleted. - * @return set of strings containing the names. This will be compared ignoring case - */ - abstract public Set getMandatoryPropertyNames(Domain domain); - - // CONSIDER: have DomainKind supply and IDomainInstance or similar - // so that it can hold instance data (e.g. a DatasetDefinition) - - /** - * Get DomainKind specific properties. - * @param domain The domain design. - * @param container Container - * @param user User - * @return Return object that holds DomainKind specific properties. - */ - abstract public @Nullable T getDomainKindProperties(GWTDomain domain, Container container, User user); - - /** - * Create a Domain appropriate for this DomainKind. - * - * @param domain The domain design. - * @param options Any domain kind specific properties/options. - * @param container Container - * @param user User - * @param forUpdate Whether the returned domain should be mutable or not - * @return The newly created Domain. - */ - abstract public Domain createDomain(GWTDomain domain, T options, Container container, User user, @Nullable TemplateInfo templateInfo, boolean forUpdate); - - /** - * Update a Domain definition appropriate for this DomainKind. - * @param original The original domain definition. - * @param update The updated domain definition. - * @param options Any domain kind specific properties/options. - * @param container Container - * @param user User - * @return A list of errors collected during the update. - */ - abstract public ValidationException updateDomain(GWTDomain original, GWTDomain update, - @Nullable T options, Container container, User user, boolean includeWarnings, @Nullable String auditUserComment); - /** - * Delete a Domain and its associated data. - * @param domain The domain to delete - */ - abstract public void deleteDomain(User user, Domain domain, @Nullable String auditUserComment); - - /** - * Get base properties defined for that domainkind. The domain parameter is only when there may be a condition - * with the particular domain that could affect the base properties (see DatasetDomainKind). Other domainkinds - * may pass through null (see AssayDomainKind). - */ - abstract public Set getBaseProperties(@Nullable Domain domain); - - /** - * Any additional properties which will get special handling in the Properties Editor. - * First use case is Lists get their property-backed primary key field added to protect it from imports and - * exclude it from exports - */ - public Set getAdditionalProtectedProperties(Domain domain) - { - return Collections.emptySet(); - } - - public Set getAdditionalProtectedPropertyNames(Domain domain) - { - Set properties = new LinkedHashSet<>(); - for (PropertyStorageSpec pss : getAdditionalProtectedProperties(domain)) - properties.add(pss.getName()); - return properties; - } - - public abstract Set getPropertyForeignKeys(Container container); - - /** - * If domains of this kind should get hard tables automatically provisioned, this returns - * the db schema where they reside. If it is null, hard tables are not to be provisioned for domains of this kind. - */ - abstract public DbScope getScope(); - abstract public String getStorageSchemaName(); - public boolean isProvisioned(Container container, String name) - { - return getStorageSchemaName() != null; - } - abstract public Set getPropertyIndices(Domain domain); - - /** - * If domain needs metadata, give the metadata schema and table names - */ - abstract public String getMetaDataSchemaName(); - abstract public String getMetaDataTableName(); - - /** - * Determines if the domain has any existing rows where the value is null for the given property - * Perhaps DomainKind should have getTableInfo() method. - */ - abstract public boolean hasNullValues(Domain domain, DomainProperty prop); - - public DbSchemaType getSchemaType() - { - return DbSchemaType.Provisioned; - } - - /** ask the domain to clear caches related to this domain */ - public void invalidate(Domain domain) - { - String schemaName = getStorageSchemaName(); - if (null == schemaName) - return; - - String storageTableName = domain.getStorageTableName(); - - if (null != storageTableName) - { - LOG.debug("Invalidating " + schemaName + "." + storageTableName); - getScope().invalidateTable(schemaName, storageTableName, getSchemaType()); - } - else - { - LOG.debug("Invalidating " + schemaName); - getScope().invalidateSchema(schemaName, getSchemaType()); - } - } - - /** - * Set of hard table names in this schema that are not provision tables - */ - abstract public Set getNonProvisionedTableNames(); - - abstract public PropertyStorageSpec getPropertySpec(PropertyDescriptor pd, Domain domain); - - /** - * @return true if we created property descriptors for base properties in the domain - */ - public boolean hasPropertiesIncludeBaseProperties() - { - return false; - } - - /** - * Default for all domain kinds is to not delete data. Lists and Datasets override this. - */ - public boolean isDeleteAllDataOnFieldImport() - { - return false; - } - - public @Nullable TableInfo getTableInfo(User user, Container container, String name, @Nullable ContainerFilter cf) - { - return null; - } - - public @Nullable TableInfo getTableInfo(User user, Container container, Domain domain, @Nullable ContainerFilter cf) - { - return getTableInfo(user, container, domain.getName(), cf); - } - - /** Called for provisioned tables after StorageProvisioner has loaded them from JDBC but before they are locked and - * cached. Use this to decorate the SchemaTableInfo with additional meta data, for example. - * - * NOTE: this is the raw-cached SchemaTableInfo, some column names may not match expected property names - * see PropertyDescriptor.getName(), PropertyDescriptor.getStorageColumnName() - */ - public void afterLoadTable(SchemaTableInfo ti, Domain domain) - { - // Most DomainKinds do nothing here - } - - /** - * Check if existing string data fits in property scale - * @param domain to execute within - * @param prop property to check - * @return true if the DomainProperty is a string and a value exists that is greater than the DomainProperty's max length - */ - public boolean exceedsMaxLength(Domain domain, DomainProperty prop) - { - //Most domains don't need to do anything here - return false; - } - - public boolean ensurePropertyLookup() - { - return false; - } - - /** - * @return true if template type in .template.xml is an instance of a module specific TemplateType pojo (which is created when - * domainTemplate.xsd is processed resulting in auto-generated TemplateType classes) - */ - public boolean matchesTemplateXML(String templateName, DomainTemplateType template, List properties) - { - return false; - } - - public boolean allowFileLinkProperties() { return false; } - public boolean allowAttachmentProperties() { return false; } - public boolean allowFlagProperties() { return true; } - public boolean allowTextChoiceProperties() { return true; } - public boolean allowMultiTextChoiceProperties() { return false; } - public boolean allowSampleSubjectProperties() { return true; } - public boolean allowTimepointProperties() { return false; } - public boolean allowUniqueConstraintProperties() { return false; } - public boolean allowCalculatedFields() { return false; } - public boolean showDefaultValueSettings() { return false; } - - public List getDisabledSystemFields(List disabledSystemFields) - { - if (disabledSystemFields == null || disabledSystemFields.isEmpty()) - return disabledSystemFields; - - Set nonDisablebleFields = getNonDisablebleFields(); - if (nonDisablebleFields == null || nonDisablebleFields.isEmpty()) - return disabledSystemFields; - - final Set disallowedFields = new CaseInsensitiveHashSet(nonDisablebleFields); - - return disabledSystemFields.stream().filter(f -> !disallowedFields.contains(f)).collect(Collectors.toList()); - } - - protected Set getNonDisablebleFields() - { - return null; - } - - public DefaultValueType[] getDefaultValueOptions(Domain domain) - { - return new DefaultValueType[] { DefaultValueType.FIXED_EDITABLE, DefaultValueType.LAST_ENTERED }; - } - - public DefaultValueType getDefaultDefaultType(Domain domain) - { - return DefaultValueType.FIXED_EDITABLE; - } - - public String getObjectUriColumnName() - { - return null; - } - - public UpdateableTableInfo.ObjectUriType getObjectUriColumn() - { - return null; - } - - /** - * Overridable validity check for domain name. Base implementation does nothing. - */ - public void validateDomainName(Container container, User user, @Nullable Domain domain, String name) - {} - - /** - * Overridable validity check. Base only executes canCreateDefinition check. - * NOTE: Due to historical limitations throws runtime exceptions instead of validation errors - * @param container being executed upon - * @param user executing service call - * @param options map to check - * @param name of design - * @param domain the existing domain object for the create/save action - * @param updatedDomainDesign the updated domain design being sent for the create/save action - */ - public void validateOptions(Container container, User user, T options, String name, Domain domain, GWTDomain updatedDomainDesign) - { - boolean isUpdate = domain != null; - - if (!isUpdate && !this.canCreateDefinition(user, container)) - throw new UnauthorizedException("You don't have permission to create a new domain"); - - if (isUpdate && !canEditDefinition(user, domain)) - throw new UnauthorizedException("You don't have permission to edit this domain"); - } - - public NameExpressionValidationResult validateNameExpressions(T options, GWTDomain domainDesign, Container container) - { - return null; - } - - /** - * @return Return preview name(s) based on the name expression configured for the designer. For DataClass, - * up to one preview names is returned. For samples, up to 2 names can be returned, with the 1st one being - * the sample preview name and the 2nd being the aliquot preview name. - */ - public @Nullable List getDomainNamePreviews(String schemaName, String queryName, Container container, User user) - { - return null; - } - - public boolean hasPermission(@NotNull UserPrincipal user, @NotNull Class perm, @NotNull UserSchema userSchema) - { - return true; - } - - // Is this domain kind considered "user managed"? e.g. list, sample type, etc - public boolean isUserCreatedType() - { - return true; - } - - public boolean supportsPhiLevel() - { - return false; - } - - public boolean supportsNamingPattern() - { - return false; - } - - public String getDomainFileDirectory() - { - return getKindName(); - } -} +/* + * Copyright (c) 2008-2019 LabKey Corporation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.labkey.api.exp.property; + +import org.apache.logging.log4j.Logger; +import org.jetbrains.annotations.NotNull; +import org.jetbrains.annotations.Nullable; +import org.labkey.api.collections.CaseInsensitiveHashSet; +import org.labkey.api.data.Container; +import org.labkey.api.data.ContainerFilter; +import org.labkey.api.data.DbSchemaType; +import org.labkey.api.data.DbScope; +import org.labkey.api.data.NameExpressionValidationResult; +import org.labkey.api.data.PropertyStorageSpec; +import org.labkey.api.data.SQLFragment; +import org.labkey.api.data.SchemaTableInfo; +import org.labkey.api.data.TableInfo; +import org.labkey.api.data.UpdateableTableInfo; +import org.labkey.api.exp.Handler; +import org.labkey.api.exp.PropertyDescriptor; +import org.labkey.api.exp.TemplateInfo; +import org.labkey.api.gwt.client.DefaultValueType; +import org.labkey.api.gwt.client.model.GWTDomain; +import org.labkey.api.gwt.client.model.GWTPropertyDescriptor; +import org.labkey.api.query.UserSchema; +import org.labkey.api.query.ValidationException; +import org.labkey.api.security.User; +import org.labkey.api.security.UserPrincipal; +import org.labkey.api.security.permissions.Permission; +import org.labkey.api.util.logging.LogHelper; +import org.labkey.api.view.ActionURL; +import org.labkey.api.view.NavTree; +import org.labkey.api.view.UnauthorizedException; +import org.labkey.api.writer.ContainerUser; +import org.labkey.data.xml.domainTemplate.DomainTemplateType; + +import java.util.Collections; +import java.util.LinkedHashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.stream.Collectors; + +abstract public class DomainKind implements Handler +{ + public static final Logger LOG = LogHelper.getLogger(DomainKind.class, "Generic domain kind activities."); + abstract public String getKindName(); + + /** + * Return a class of DomainKind's bean which carries domain specific properties. + * This class will be used when marshalling/unmarshalling via Jackson during Create and Save/Update Domain + * @return Class of DomainKind's bean with domain specific properties + */ + abstract public Class getTypeClass(); + + /** + * Give the domain kind a chance to process / map the arguments before converting it to the DomainKind properties object. + * @param arguments initial arguments coming from the caller + * @return updated arguments + */ + public Map processArguments(Container container, User user, Map arguments) + { + return arguments; + } + + abstract public String getTypeLabel(Domain domain); + abstract public SQLFragment sqlObjectIdsInDomain(Domain domain); + + /** + * Create a DomainURI for a Domain that may or may not exist yet. + */ + abstract public String generateDomainURI(String schemaName, String queryName, Container container, User user); + + abstract public @Nullable ActionURL urlShowData(Domain domain, ContainerUser containerUser); + abstract public @Nullable ActionURL urlEditDefinition(Domain domain, ContainerUser containerUser); + abstract public ActionURL urlCreateDefinition(String schemaName, String queryName, Container container, User user); + + // Override to return a non-null String and the generic domain editor will display it in an "Instructions" webpart above the field properties + public @Nullable String getDomainEditorInstructions() + { + return null; + } + + abstract public boolean canCreateDefinition(User user, Container container); + + abstract public boolean canEditDefinition(User user, Domain domain); + + abstract public boolean canDeleteDefinition(User user, Domain domain); + + // Override to customize the nav trail on shared pages like edit domain + abstract public void addNavTrail(NavTree root, Container c, User user); + + // Do any special handling before a PropertyDescriptor is deleted -- do nothing by default + abstract public void deletePropertyDescriptor(Domain domain, User user, PropertyDescriptor pd); + + /** + * Return the set of names that should not be allowed for properties. E.g. + * the names of columns from the hard table underlying this type + * @return set of strings containing the names. This will be compared ignoring case + */ + abstract public Set getReservedPropertyNames(Domain domain, User user); + + public Set getReservedPropertyNames(Domain domain, User user, boolean forCreate) + { + return getReservedPropertyNames(domain, user); + } + + public Set getReservedPropertyNamePrefixes() + { + return Collections.emptySet(); + } + + /** + * Return the set of names that are always required and cannot subsequently + * be deleted. + * @return set of strings containing the names. This will be compared ignoring case + */ + abstract public Set getMandatoryPropertyNames(Domain domain); + + // CONSIDER: have DomainKind supply and IDomainInstance or similar + // so that it can hold instance data (e.g. a DatasetDefinition) + + /** + * Get DomainKind specific properties. + * @param domain The domain design. + * @param container Container + * @param user User + * @return Return object that holds DomainKind specific properties. + */ + abstract public @Nullable T getDomainKindProperties(GWTDomain domain, Container container, User user); + + /** + * Create a Domain appropriate for this DomainKind. + * + * @param domain The domain design. + * @param options Any domain kind specific properties/options. + * @param container Container + * @param user User + * @param forUpdate Whether the returned domain should be mutable or not + * @return The newly created Domain. + */ + abstract public Domain createDomain(GWTDomain domain, T options, Container container, User user, @Nullable TemplateInfo templateInfo, boolean forUpdate); + + /** + * Update a Domain definition appropriate for this DomainKind. + * @param original The original domain definition. + * @param update The updated domain definition. + * @param options Any domain kind specific properties/options. + * @param container Container + * @param user User + * @return A list of errors collected during the update. + */ + abstract public ValidationException updateDomain(GWTDomain original, GWTDomain update, + @Nullable T options, Container container, User user, boolean includeWarnings, @Nullable String auditUserComment); + /** + * Delete a Domain and its associated data. + * @param domain The domain to delete + */ + abstract public void deleteDomain(User user, Domain domain, @Nullable String auditUserComment); + + /** + * Get base properties defined for that domainkind. The domain parameter is only when there may be a condition + * with the particular domain that could affect the base properties (see DatasetDomainKind). Other domainkinds + * may pass through null (see AssayDomainKind). + */ + abstract public Set getBaseProperties(@Nullable Domain domain); + + /** + * Any additional properties which will get special handling in the Properties Editor. + * First use case is Lists get their property-backed primary key field added to protect it from imports and + * exclude it from exports + */ + public Set getAdditionalProtectedProperties(Domain domain) + { + return Collections.emptySet(); + } + + public Set getAdditionalProtectedPropertyNames(Domain domain) + { + Set properties = new LinkedHashSet<>(); + for (PropertyStorageSpec pss : getAdditionalProtectedProperties(domain)) + properties.add(pss.getName()); + return properties; + } + + public abstract Set getPropertyForeignKeys(Container container); + + /** + * If domains of this kind should get hard tables automatically provisioned, this returns + * the db schema where they reside. If it is null, hard tables are not to be provisioned for domains of this kind. + */ + abstract public DbScope getScope(); + abstract public String getStorageSchemaName(); + public boolean isProvisioned(Container container, String name) + { + return getStorageSchemaName() != null; + } + abstract public Set getPropertyIndices(Domain domain); + + /** + * If domain needs metadata, give the metadata schema and table names + */ + abstract public String getMetaDataSchemaName(); + abstract public String getMetaDataTableName(); + + /** + * Determines if the domain has any existing rows where the value is null for the given property + * Perhaps DomainKind should have getTableInfo() method. + */ + abstract public boolean hasNullValues(Domain domain, DomainProperty prop); + + public DbSchemaType getSchemaType() + { + return DbSchemaType.Provisioned; + } + + /** ask the domain to clear caches related to this domain */ + public void invalidate(Domain domain) + { + String schemaName = getStorageSchemaName(); + if (null == schemaName) + return; + + String storageTableName = domain.getStorageTableName(); + + if (null != storageTableName) + { + LOG.debug("Invalidating " + schemaName + "." + storageTableName); + getScope().invalidateTable(schemaName, storageTableName, getSchemaType()); + } + else + { + LOG.debug("Invalidating " + schemaName); + getScope().invalidateSchema(schemaName, getSchemaType()); + } + } + + /** + * Set of hard table names in this schema that are not provision tables + */ + abstract public Set getNonProvisionedTableNames(); + + abstract public PropertyStorageSpec getPropertySpec(PropertyDescriptor pd, Domain domain); + + /** + * @return true if we created property descriptors for base properties in the domain + */ + public boolean hasPropertiesIncludeBaseProperties() + { + return false; + } + + /** + * Default for all domain kinds is to not delete data. Lists and Datasets override this. + */ + public boolean isDeleteAllDataOnFieldImport() + { + return false; + } + + public @Nullable TableInfo getTableInfo(User user, Container container, String name, @Nullable ContainerFilter cf) + { + return null; + } + + public @Nullable TableInfo getTableInfo(User user, Container container, Domain domain, @Nullable ContainerFilter cf) + { + return getTableInfo(user, container, domain.getName(), cf); + } + + /** Called for provisioned tables after StorageProvisioner has loaded them from JDBC but before they are locked and + * cached. Use this to decorate the SchemaTableInfo with additional meta data, for example. + * + * NOTE: this is the raw-cached SchemaTableInfo, some column names may not match expected property names + * see PropertyDescriptor.getName(), PropertyDescriptor.getStorageColumnName() + */ + public void afterLoadTable(SchemaTableInfo ti, Domain domain) + { + // Most DomainKinds do nothing here + } + + /** + * Check if existing string data fits in property scale + * @param domain to execute within + * @param prop property to check + * @return true if the DomainProperty is a string and a value exists that is greater than the DomainProperty's max length + */ + public boolean exceedsMaxLength(Domain domain, DomainProperty prop) + { + //Most domains don't need to do anything here + return false; + } + + public boolean ensurePropertyLookup() + { + return false; + } + + /** + * @return true if template type in .template.xml is an instance of a module specific TemplateType pojo (which is created when + * domainTemplate.xsd is processed resulting in auto-generated TemplateType classes) + */ + public boolean matchesTemplateXML(String templateName, DomainTemplateType template, List properties) + { + return false; + } + + public boolean allowFileLinkProperties() { return false; } + public boolean allowAttachmentProperties() { return false; } + public boolean allowFlagProperties() { return true; } + public boolean allowTextChoiceProperties() { return true; } + public boolean allowMultiChoiceProperties() { return false; } + public boolean allowSampleSubjectProperties() { return true; } + public boolean allowTimepointProperties() { return false; } + public boolean allowUniqueConstraintProperties() { return false; } + public boolean allowCalculatedFields() { return false; } + public boolean showDefaultValueSettings() { return false; } + + public List getDisabledSystemFields(List disabledSystemFields) + { + if (disabledSystemFields == null || disabledSystemFields.isEmpty()) + return disabledSystemFields; + + Set nonDisablebleFields = getNonDisablebleFields(); + if (nonDisablebleFields == null || nonDisablebleFields.isEmpty()) + return disabledSystemFields; + + final Set disallowedFields = new CaseInsensitiveHashSet(nonDisablebleFields); + + return disabledSystemFields.stream().filter(f -> !disallowedFields.contains(f)).collect(Collectors.toList()); + } + + protected Set getNonDisablebleFields() + { + return null; + } + + public DefaultValueType[] getDefaultValueOptions(Domain domain) + { + return new DefaultValueType[] { DefaultValueType.FIXED_EDITABLE, DefaultValueType.LAST_ENTERED }; + } + + public DefaultValueType getDefaultDefaultType(Domain domain) + { + return DefaultValueType.FIXED_EDITABLE; + } + + public String getObjectUriColumnName() + { + return null; + } + + public UpdateableTableInfo.ObjectUriType getObjectUriColumn() + { + return null; + } + + /** + * Overridable validity check for domain name. Base implementation does nothing. + */ + public void validateDomainName(Container container, User user, @Nullable Domain domain, String name) + {} + + /** + * Overridable validity check. Base only executes canCreateDefinition check. + * NOTE: Due to historical limitations throws runtime exceptions instead of validation errors + * @param container being executed upon + * @param user executing service call + * @param options map to check + * @param name of design + * @param domain the existing domain object for the create/save action + * @param updatedDomainDesign the updated domain design being sent for the create/save action + */ + public void validateOptions(Container container, User user, T options, String name, Domain domain, GWTDomain updatedDomainDesign) + { + boolean isUpdate = domain != null; + + if (!isUpdate && !this.canCreateDefinition(user, container)) + throw new UnauthorizedException("You don't have permission to create a new domain"); + + if (isUpdate && !canEditDefinition(user, domain)) + throw new UnauthorizedException("You don't have permission to edit this domain"); + } + + public NameExpressionValidationResult validateNameExpressions(T options, GWTDomain domainDesign, Container container) + { + return null; + } + + /** + * @return Return preview name(s) based on the name expression configured for the designer. For DataClass, + * up to one preview names is returned. For samples, up to 2 names can be returned, with the 1st one being + * the sample preview name and the 2nd being the aliquot preview name. + */ + public @Nullable List getDomainNamePreviews(String schemaName, String queryName, Container container, User user) + { + return null; + } + + public boolean hasPermission(@NotNull UserPrincipal user, @NotNull Class perm, @NotNull UserSchema userSchema) + { + return true; + } + + // Is this domain kind considered "user managed"? e.g. list, sample type, etc + public boolean isUserCreatedType() + { + return true; + } + + public boolean supportsPhiLevel() + { + return false; + } + + public boolean supportsNamingPattern() + { + return false; + } + + public String getDomainFileDirectory() + { + return getKindName(); + } +} diff --git a/api/src/org/labkey/api/exp/property/DomainUtil.java b/api/src/org/labkey/api/exp/property/DomainUtil.java index af2309f42c8..cf9f540bf25 100644 --- a/api/src/org/labkey/api/exp/property/DomainUtil.java +++ b/api/src/org/labkey/api/exp/property/DomainUtil.java @@ -35,6 +35,7 @@ import org.labkey.api.data.ContainerFilter; import org.labkey.api.data.ContainerManager; import org.labkey.api.data.ContainerService; +import org.labkey.api.data.CoreSchema; import org.labkey.api.data.NameGenerator; import org.labkey.api.data.PHI; import org.labkey.api.data.PropertyStorageSpec; @@ -78,6 +79,8 @@ import org.labkey.api.query.UserSchema; import org.labkey.api.query.ValidationException; import org.labkey.api.security.User; +import org.labkey.api.settings.AppProps; +import org.labkey.api.settings.OptionalFeatureService; import org.labkey.api.util.DateUtil; import org.labkey.api.util.GUID; import org.labkey.api.util.JdbcUtil; @@ -428,6 +431,15 @@ public static List getCalculatedFieldsForDefaultView(@NotNull TableInf return calculatedFieldKeys; } + public static boolean allowMultiChoice(DomainKind kind) + { + if (!kind.allowMultiChoiceProperties()) + return false; + if (!OptionalFeatureService.get().isFeatureEnabled(AppProps.MULTI_VALUE_TEXT_CHOICE)) + return false; + return CoreSchema.getInstance().getSqlDialect().isPostgreSQL(); + } + private static GWTDomain getDomain(Domain dd) { GWTDomain gwtDomain = new GWTDomain<>(); @@ -447,6 +459,7 @@ private static GWTDomain getDomain(Domain dd) gwtDomain.setAllowFileLinkProperties(kind.allowFileLinkProperties()); gwtDomain.setAllowFlagProperties(kind.allowFlagProperties()); gwtDomain.setAllowTextChoiceProperties(kind.allowTextChoiceProperties()); + gwtDomain.setAllowMultiChoiceProperties(allowMultiChoice(kind)); gwtDomain.setAllowSampleSubjectProperties(kind.allowSampleSubjectProperties()); gwtDomain.setAllowTimepointProperties(kind.allowTimepointProperties()); gwtDomain.setAllowUniqueConstraintProperties(kind.allowUniqueConstraintProperties()); @@ -466,6 +479,7 @@ public static GWTDomain getTemplateDomainForDomainKind(Do gwtDomain.setAllowFileLinkProperties(kind.allowFileLinkProperties()); gwtDomain.setAllowFlagProperties(kind.allowFlagProperties()); gwtDomain.setAllowTextChoiceProperties(kind.allowTextChoiceProperties()); + gwtDomain.setAllowMultiChoiceProperties(allowMultiChoice(kind)); gwtDomain.setAllowSampleSubjectProperties(kind.allowSampleSubjectProperties()); gwtDomain.setAllowTimepointProperties(kind.allowTimepointProperties()); gwtDomain.setShowDefaultValueSettings(kind.showDefaultValueSettings()); diff --git a/api/src/org/labkey/api/reader/TabLoader.java b/api/src/org/labkey/api/reader/TabLoader.java index 865abbf6091..7ebf7585b8b 100644 --- a/api/src/org/labkey/api/reader/TabLoader.java +++ b/api/src/org/labkey/api/reader/TabLoader.java @@ -26,6 +26,7 @@ import org.junit.Assert; import org.junit.Test; import org.labkey.api.data.Container; +import org.labkey.api.data.MultiChoice; import org.labkey.api.dataiterator.HashDataIterator; import org.labkey.api.iterator.BeanIterator; import org.labkey.api.iterator.CloseableIterator; @@ -405,6 +406,16 @@ private String[] readFields(TabBufferedReader r, @Nullable ColumnDescriptor[] co char ch = buf.charAt(start); char chQuote = '"'; + boolean isArrayColumn = false; + if (columns != null) + { + var column = columns[colIndex]; + if (column != null) + isArrayColumn = column.clazz == MultiChoice.class || column.clazz == MultiChoice.Array.class; + } + + boolean parseEnclosedQuotes = _parseEnclosedQuotes || isArrayColumn; + colIndex++; boolean isDelimiterOrQuote = false; @@ -431,7 +442,7 @@ else if (ch == chQuote) if (nextLine == null) { // We've reached the end of the input, so there's nothing else to append - if (_parseEnclosedQuotes) + if (parseEnclosedQuotes) isDelimiterOrQuote = false; break; } @@ -446,7 +457,7 @@ else if (ch == chQuote) // " a, " b should be parsed as [" a, " b], not [a, b] // if the next quote is before the end of the buffer and the next non-blank character is not the delimiter, // retain the quote as a mid-field value. - if (_parseEnclosedQuotes && end != buf.length() - 1 && (fieldEnd == -1 || !_whitespacePattern.matcher(buf.substring(end+1, fieldEnd)).matches())) + if (parseEnclosedQuotes && end != buf.length() - 1 && (fieldEnd == -1 || !_whitespacePattern.matcher(buf.substring(end+1, fieldEnd)).matches())) isDelimiterOrQuote = false; break; } diff --git a/api/src/org/labkey/api/settings/AppProps.java b/api/src/org/labkey/api/settings/AppProps.java index 6395044aeb2..0ed0dc058c3 100644 --- a/api/src/org/labkey/api/settings/AppProps.java +++ b/api/src/org/labkey/api/settings/AppProps.java @@ -1,267 +1,268 @@ -/* - * Copyright (c) 2008-2019 LabKey Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.labkey.api.settings; - -import jakarta.servlet.http.HttpServletRequest; -import org.jetbrains.annotations.NotNull; -import org.jetbrains.annotations.Nullable; -import org.labkey.api.data.ContainerManager; -import org.labkey.api.module.DefaultModule; -import org.labkey.api.module.SupportedDatabase; -import org.labkey.api.util.ExceptionReportingLevel; -import org.labkey.api.util.Path; -import org.labkey.api.util.UsageReportingLevel; -import org.labkey.api.view.ActionURL; - -import java.io.File; -import java.util.List; -import java.util.Map; -import java.util.Set; - -/** - * Stores basic site-wide configuration. - * @see org.labkey.api.settings.WriteableAppProps - */ -public interface AppProps -{ - AppProps _instance = new AppPropsImpl(); - - String SCOPE_SITE_SETTINGS = "SiteSettings"; - - // Used for all optional features; "experimental" for historical reasons. - String OPTIONAL_FEATURE_PREFIX = "experimentalFeature."; - String SCOPE_OPTIONAL_FEATURE = "ExperimentalFeature"; // Startup property prefix for all optional features; "Experimental" for historical reasons. - String EXPERIMENTAL_NO_GUESTS = "disableGuestAccount"; - String EXPERIMENTAL_BLOCKER = "blockMaliciousClients"; - String EXPERIMENTAL_RESOLVE_PROPERTY_URI_COLUMNS = "resolve-property-uri-columns"; - String ADMIN_PROVIDED_ALLOWED_EXTERNAL_RESOURCES = "allowedExternalResources"; - String QUANTITY_COLUMN_SUFFIX_TESTING = "quantityColumnSuffixTesting"; - String GENERATE_CONTROLLER_FIRST_URLS = "generateControllerFirstUrls"; - String REJECT_CONTROLLER_FIRST_URLS = "rejectControllerFirstUrls"; - - String UNKNOWN_VERSION = "Unknown Release Version"; - - static AppProps getInstance() - { - return _instance; - } - - static WriteableAppProps getWriteableInstance() - { - return new WriteableAppProps(ContainerManager.getRoot()); - } - - String getServerSessionGUID(); - - boolean isMailRecorderEnabled(); - - boolean isOptionalFeatureEnabled(String feature); - - boolean isDevMode(); - - @Nullable - String getEnlistmentId(); - - boolean isCachingAllowed(); - - boolean isRecompileJspEnabled(); - - /** - * Indicates whether modules' "sourcePath" and "buildPath" values be ignored. This allows a server to run in devMode - * without the risk of loading unwanted resources from a source tree that may not match the deployed server. - * - * WARNING: Setting this flag will interfere with the population of module beans, resulting in a - * mismatch between deployed modules and their properties on the server. - * - * @return value of the 'labkey.ignoreModuleSource' system property. Defaults to false - * - * @see org.labkey.api.module.DefaultModule#setSourcePath(String) - * @see org.labkey.api.module.DefaultModule#setBuildPath(String) - * @see DefaultModule#computeResourceDirectory() - */ - boolean isIgnoreModuleSource(); - - void setProjectRoot(String projectRoot); - - /** - * @return the root of the main source tree - */ - @Nullable - String getProjectRoot(); - - /** - * @return directory under which all containers will automatically have their own subdirectory for storing files - */ - @Nullable - File getFileSystemRoot(); - - @NotNull - UsageReportingLevel getUsageReportingLevel(); - - /** - * Returns the core module's release version, a string such as "20.3-SNAPSHOT", "20.1.0", or "20.3.7". - * Or "Unknown Release Version". - */ - @NotNull - String getReleaseVersion(); - - /** - * Convenience method for getting the core schema version, returning 0.0 instead of null - */ - double getSchemaVersion(); - - String getContextPath(); - - Path getParsedContextPath(); - - int getServerPort(); - - String getScheme(); - - String getServerName(); - - /** - * Save the current request URL if the base server URL property is not set - */ - void ensureBaseServerUrl(HttpServletRequest request); - - void setContextPath(String contextPath); - - boolean isSetBaseServerUrl(); - - String getBaseServerUrl(); - - String getHomePageUrl(); - - ActionURL getHomePageActionURL(); - - String getSiteWelcomePageUrlString(); - - int getLookAndFeelRevision(); - - String getDefaultLsidAuthority(); - - String getPipelineToolsDirectory(); - - boolean isSSLRequired(); - - boolean isUserRequestedAdminOnlyMode(); - - String getAdminOnlyMessage(); - - boolean isShowRibbonMessage(); - - @Nullable String getRibbonMessage(); - - int getSSLPort(); - - int getMemoryUsageDumpInterval(); - - /** Timeout in seconds for read-only HTTP requests, after which resources like DB connections and spawned processes will be killed. Set to 0 to disable. */ - int getReadOnlyHttpRequestTimeout(); - - int getMaxBLOBSize(); - - boolean isExt3Required(); - - boolean isExt3APIRequired(); - - ExceptionReportingLevel getExceptionReportingLevel(); - - /** - * Flag specifying if the project navigation access is open/closed. Open (default) means users will see the full - * folder tree for all folders they have permissions to see. Closed follows the rules as specified in issue #32718. - * - * @return if navigation access is open - */ - boolean isNavigationAccessOpen(); - - boolean isSelfReportExceptions(); - - String getServerGUID(); - - String getBLASTServerBaseURL(); - - /** @return the name of the Tomcat XML deployment descriptor based on the context path for this install - now always application.properties */ - String getWebappConfigurationFilename(); - - /** - * Email address of the primary site or application administrator, set on the site settings page. Useful in error - * messages when only an administrator can help. Returns null if there are no site or application admins (i.e., - * only impersonating troubleshooters). - * - * @return Email address of the primary site or application administrator - */ - @Nullable String getAdministratorContactEmail(boolean includeAppAdmins); - - /** - * Whether to use the newer, and preferred, container-relative style of URLs of the form - * /contextPath/container/controller-action.view, or the older controller-relative style like - * /contextPath/controller/container/action.view - */ - boolean getUseContainerRelativeURL(); - - boolean isAllowApiKeys(); - - int getApiKeyExpirationSeconds(); - - boolean isAllowSessionKeys(); - - // configurable http security settings - - /** - * @return "SAMEORIGIN" or "DENY" or "ALLOW" - */ - String getXFrameOption(); - - String getStaticFilesPrefix(); - - boolean isWebfilesRootEnabled(); - - boolean isFileUploadDisabled(); - - boolean isInvalidFilenameUploadBlocked(); - - boolean isInvalidFilenameBlocked(); - - /** @return whether the server should include its name and version as a header in HTTP responses */ - boolean isIncludeServerHttpHeader(); - - /** - * @return List of configured external redirect hosts - */ - @NotNull - List getExternalRedirectHosts(); - - /** - * @return List of configured external resource hosts - */ - @Deprecated // Left for upgrade code only - @NotNull - List getExternalSourceHosts(); - - Map getStashedStartupProperties(); - - @NotNull String getDistributionName(); - - @NotNull String getDistributionFilename(); - - @NotNull Set getDistributionSupportedDatabases(); - - @NotNull List getAllowedExtensions(); - - @NotNull String getAllowedExternalResourceHosts(); -} +/* + * Copyright (c) 2008-2019 LabKey Corporation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.labkey.api.settings; + +import jakarta.servlet.http.HttpServletRequest; +import org.jetbrains.annotations.NotNull; +import org.jetbrains.annotations.Nullable; +import org.labkey.api.data.ContainerManager; +import org.labkey.api.module.DefaultModule; +import org.labkey.api.module.SupportedDatabase; +import org.labkey.api.util.ExceptionReportingLevel; +import org.labkey.api.util.Path; +import org.labkey.api.util.UsageReportingLevel; +import org.labkey.api.view.ActionURL; + +import java.io.File; +import java.util.List; +import java.util.Map; +import java.util.Set; + +/** + * Stores basic site-wide configuration. + * @see org.labkey.api.settings.WriteableAppProps + */ +public interface AppProps +{ + AppProps _instance = new AppPropsImpl(); + + String SCOPE_SITE_SETTINGS = "SiteSettings"; + + // Used for all optional features; "experimental" for historical reasons. + String OPTIONAL_FEATURE_PREFIX = "experimentalFeature."; + String SCOPE_OPTIONAL_FEATURE = "ExperimentalFeature"; // Startup property prefix for all optional features; "Experimental" for historical reasons. + String EXPERIMENTAL_NO_GUESTS = "disableGuestAccount"; + String EXPERIMENTAL_BLOCKER = "blockMaliciousClients"; + String EXPERIMENTAL_RESOLVE_PROPERTY_URI_COLUMNS = "resolve-property-uri-columns"; + String ADMIN_PROVIDED_ALLOWED_EXTERNAL_RESOURCES = "allowedExternalResources"; + String QUANTITY_COLUMN_SUFFIX_TESTING = "quantityColumnSuffixTesting"; + String GENERATE_CONTROLLER_FIRST_URLS = "generateControllerFirstUrls"; + String REJECT_CONTROLLER_FIRST_URLS = "rejectControllerFirstUrls"; + String MULTI_VALUE_TEXT_CHOICE = "multiChoiceDataType"; + + String UNKNOWN_VERSION = "Unknown Release Version"; + + static AppProps getInstance() + { + return _instance; + } + + static WriteableAppProps getWriteableInstance() + { + return new WriteableAppProps(ContainerManager.getRoot()); + } + + String getServerSessionGUID(); + + boolean isMailRecorderEnabled(); + + boolean isOptionalFeatureEnabled(String feature); + + boolean isDevMode(); + + @Nullable + String getEnlistmentId(); + + boolean isCachingAllowed(); + + boolean isRecompileJspEnabled(); + + /** + * Indicates whether modules' "sourcePath" and "buildPath" values be ignored. This allows a server to run in devMode + * without the risk of loading unwanted resources from a source tree that may not match the deployed server. + * + * WARNING: Setting this flag will interfere with the population of module beans, resulting in a + * mismatch between deployed modules and their properties on the server. + * + * @return value of the 'labkey.ignoreModuleSource' system property. Defaults to false + * + * @see org.labkey.api.module.DefaultModule#setSourcePath(String) + * @see org.labkey.api.module.DefaultModule#setBuildPath(String) + * @see DefaultModule#computeResourceDirectory() + */ + boolean isIgnoreModuleSource(); + + void setProjectRoot(String projectRoot); + + /** + * @return the root of the main source tree + */ + @Nullable + String getProjectRoot(); + + /** + * @return directory under which all containers will automatically have their own subdirectory for storing files + */ + @Nullable + File getFileSystemRoot(); + + @NotNull + UsageReportingLevel getUsageReportingLevel(); + + /** + * Returns the core module's release version, a string such as "20.3-SNAPSHOT", "20.1.0", or "20.3.7". + * Or "Unknown Release Version". + */ + @NotNull + String getReleaseVersion(); + + /** + * Convenience method for getting the core schema version, returning 0.0 instead of null + */ + double getSchemaVersion(); + + String getContextPath(); + + Path getParsedContextPath(); + + int getServerPort(); + + String getScheme(); + + String getServerName(); + + /** + * Save the current request URL if the base server URL property is not set + */ + void ensureBaseServerUrl(HttpServletRequest request); + + void setContextPath(String contextPath); + + boolean isSetBaseServerUrl(); + + String getBaseServerUrl(); + + String getHomePageUrl(); + + ActionURL getHomePageActionURL(); + + String getSiteWelcomePageUrlString(); + + int getLookAndFeelRevision(); + + String getDefaultLsidAuthority(); + + String getPipelineToolsDirectory(); + + boolean isSSLRequired(); + + boolean isUserRequestedAdminOnlyMode(); + + String getAdminOnlyMessage(); + + boolean isShowRibbonMessage(); + + @Nullable String getRibbonMessage(); + + int getSSLPort(); + + int getMemoryUsageDumpInterval(); + + /** Timeout in seconds for read-only HTTP requests, after which resources like DB connections and spawned processes will be killed. Set to 0 to disable. */ + int getReadOnlyHttpRequestTimeout(); + + int getMaxBLOBSize(); + + boolean isExt3Required(); + + boolean isExt3APIRequired(); + + ExceptionReportingLevel getExceptionReportingLevel(); + + /** + * Flag specifying if the project navigation access is open/closed. Open (default) means users will see the full + * folder tree for all folders they have permissions to see. Closed follows the rules as specified in issue #32718. + * + * @return if navigation access is open + */ + boolean isNavigationAccessOpen(); + + boolean isSelfReportExceptions(); + + String getServerGUID(); + + String getBLASTServerBaseURL(); + + /** @return the name of the Tomcat XML deployment descriptor based on the context path for this install - now always application.properties */ + String getWebappConfigurationFilename(); + + /** + * Email address of the primary site or application administrator, set on the site settings page. Useful in error + * messages when only an administrator can help. Returns null if there are no site or application admins (i.e., + * only impersonating troubleshooters). + * + * @return Email address of the primary site or application administrator + */ + @Nullable String getAdministratorContactEmail(boolean includeAppAdmins); + + /** + * Whether to use the newer, and preferred, container-relative style of URLs of the form + * /contextPath/container/controller-action.view, or the older controller-relative style like + * /contextPath/controller/container/action.view + */ + boolean getUseContainerRelativeURL(); + + boolean isAllowApiKeys(); + + int getApiKeyExpirationSeconds(); + + boolean isAllowSessionKeys(); + + // configurable http security settings + + /** + * @return "SAMEORIGIN" or "DENY" or "ALLOW" + */ + String getXFrameOption(); + + String getStaticFilesPrefix(); + + boolean isWebfilesRootEnabled(); + + boolean isFileUploadDisabled(); + + boolean isInvalidFilenameUploadBlocked(); + + boolean isInvalidFilenameBlocked(); + + /** @return whether the server should include its name and version as a header in HTTP responses */ + boolean isIncludeServerHttpHeader(); + + /** + * @return List of configured external redirect hosts + */ + @NotNull + List getExternalRedirectHosts(); + + /** + * @return List of configured external resource hosts + */ + @Deprecated // Left for upgrade code only + @NotNull + List getExternalSourceHosts(); + + Map getStashedStartupProperties(); + + @NotNull String getDistributionName(); + + @NotNull String getDistributionFilename(); + + @NotNull Set getDistributionSupportedDatabases(); + + @NotNull List getAllowedExtensions(); + + @NotNull String getAllowedExternalResourceHosts(); +} diff --git a/experiment/src/org/labkey/experiment/ExperimentModule.java b/experiment/src/org/labkey/experiment/ExperimentModule.java index ab459df94f7..fb9cb2d52a4 100644 --- a/experiment/src/org/labkey/experiment/ExperimentModule.java +++ b/experiment/src/org/labkey/experiment/ExperimentModule.java @@ -1,1128 +1,1131 @@ -/* - * Copyright (c) 2008-2019 LabKey Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.labkey.experiment; - -import org.apache.commons.lang3.math.NumberUtils; -import org.jetbrains.annotations.NotNull; -import org.jetbrains.annotations.Nullable; -import org.labkey.api.admin.FolderSerializationRegistry; -import org.labkey.api.assay.AssayProvider; -import org.labkey.api.assay.AssayService; -import org.labkey.api.attachments.AttachmentService; -import org.labkey.api.audit.AuditLogService; -import org.labkey.api.audit.SampleTimelineAuditEvent; -import org.labkey.api.collections.LongHashMap; -import org.labkey.api.data.Container; -import org.labkey.api.data.ContainerFilter; -import org.labkey.api.data.ContainerManager; -import org.labkey.api.data.CoreSchema; -import org.labkey.api.data.DbSchema; -import org.labkey.api.data.DbSchemaType; -import org.labkey.api.data.JdbcType; -import org.labkey.api.data.NameGenerator; -import org.labkey.api.data.SQLFragment; -import org.labkey.api.data.SimpleFilter; -import org.labkey.api.data.SimpleFilter.FilterClause; -import org.labkey.api.data.SqlSelector; -import org.labkey.api.data.TableInfo; -import org.labkey.api.data.TableSelector; -import org.labkey.api.data.UpgradeCode; -import org.labkey.api.defaults.DefaultValueService; -import org.labkey.api.exp.ExperimentException; -import org.labkey.api.exp.ExperimentRunType; -import org.labkey.api.exp.Lsid; -import org.labkey.api.exp.OntologyManager; -import org.labkey.api.exp.PropertyType; -import org.labkey.api.exp.api.DefaultExperimentDataHandler; -import org.labkey.api.exp.api.ExpData; -import org.labkey.api.exp.api.ExpDataClass; -import org.labkey.api.exp.api.ExpLineageService; -import org.labkey.api.exp.api.ExpMaterial; -import org.labkey.api.exp.api.ExpProtocol; -import org.labkey.api.exp.api.ExpProtocolAttachmentType; -import org.labkey.api.exp.api.ExpRunAttachmentType; -import org.labkey.api.exp.api.ExpSampleType; -import org.labkey.api.exp.api.ExperimentJSONConverter; -import org.labkey.api.exp.api.ExperimentService; -import org.labkey.api.exp.api.FilterProtocolInputCriteria; -import org.labkey.api.exp.api.SampleTypeDomainKind; -import org.labkey.api.exp.api.SampleTypeService; -import org.labkey.api.exp.api.StorageProvisioner; -import org.labkey.api.exp.property.DomainAuditProvider; -import org.labkey.api.exp.property.DomainPropertyAuditProvider; -import org.labkey.api.exp.property.ExperimentProperty; -import org.labkey.api.exp.property.PropertyService; -import org.labkey.api.exp.property.SystemProperty; -import org.labkey.api.exp.query.ExpDataClassTable; -import org.labkey.api.exp.query.ExpSampleTypeTable; -import org.labkey.api.exp.query.ExpSchema; -import org.labkey.api.exp.query.SamplesSchema; -import org.labkey.api.exp.xar.LSIDRelativizer; -import org.labkey.api.exp.xar.LsidUtils; -import org.labkey.api.files.FileContentService; -import org.labkey.api.files.TableUpdaterFileListener; -import org.labkey.api.migration.DatabaseMigrationService; -import org.labkey.api.migration.ExperimentDeleteService; -import org.labkey.api.migration.MigrationTableHandler; -import org.labkey.api.module.ModuleContext; -import org.labkey.api.module.ModuleLoader; -import org.labkey.api.module.SpringModule; -import org.labkey.api.module.Summary; -import org.labkey.api.ontology.OntologyService; -import org.labkey.api.ontology.Quantity; -import org.labkey.api.ontology.Unit; -import org.labkey.api.pipeline.PipelineService; -import org.labkey.api.query.FieldKey; -import org.labkey.api.query.FilteredTable; -import org.labkey.api.query.QueryService; -import org.labkey.api.query.UserSchema; -import org.labkey.api.search.SearchService; -import org.labkey.api.security.User; -import org.labkey.api.security.roles.RoleManager; -import org.labkey.api.settings.AppProps; -import org.labkey.api.settings.OptionalFeatureService; -import org.labkey.api.usageMetrics.UsageMetricsService; -import org.labkey.api.util.GUID; -import org.labkey.api.util.JspTestCase; -import org.labkey.api.util.PageFlowUtil; -import org.labkey.api.util.StringUtilsLabKey; -import org.labkey.api.util.SystemMaintenance; -import org.labkey.api.view.AlwaysAvailableWebPartFactory; -import org.labkey.api.view.BaseWebPartFactory; -import org.labkey.api.view.HttpView; -import org.labkey.api.view.JspView; -import org.labkey.api.view.Portal; -import org.labkey.api.view.ViewContext; -import org.labkey.api.view.WebPartFactory; -import org.labkey.api.view.WebPartView; -import org.labkey.api.view.template.WarningService; -import org.labkey.api.vocabulary.security.DesignVocabularyPermission; -import org.labkey.api.webdav.WebdavResource; -import org.labkey.api.webdav.WebdavService; -import org.labkey.experiment.api.DataClassDomainKind; -import org.labkey.experiment.api.ExpDataClassImpl; -import org.labkey.experiment.api.ExpDataClassTableImpl; -import org.labkey.experiment.api.ExpDataClassType; -import org.labkey.experiment.api.ExpDataImpl; -import org.labkey.experiment.api.ExpDataTableImpl; -import org.labkey.experiment.api.ExpMaterialImpl; -import org.labkey.experiment.api.ExpProtocolImpl; -import org.labkey.experiment.api.ExpSampleTypeImpl; -import org.labkey.experiment.api.ExpSampleTypeTableImpl; -import org.labkey.experiment.api.ExperimentServiceImpl; -import org.labkey.experiment.api.ExperimentStressTest; -import org.labkey.experiment.api.GraphAlgorithms; -import org.labkey.experiment.api.LineageTest; -import org.labkey.experiment.api.LogDataType; -import org.labkey.experiment.api.Protocol; -import org.labkey.experiment.api.SampleTypeServiceImpl; -import org.labkey.experiment.api.SampleTypeUpdateServiceDI; -import org.labkey.experiment.api.UniqueValueCounterTestCase; -import org.labkey.experiment.api.VocabularyDomainKind; -import org.labkey.experiment.api.data.ChildOfCompareType; -import org.labkey.experiment.api.data.ChildOfMethod; -import org.labkey.experiment.api.data.LineageCompareType; -import org.labkey.experiment.api.data.ParentOfCompareType; -import org.labkey.experiment.api.data.ParentOfMethod; -import org.labkey.experiment.api.property.DomainImpl; -import org.labkey.experiment.api.property.DomainPropertyImpl; -import org.labkey.experiment.api.property.LengthValidator; -import org.labkey.experiment.api.property.LookupValidator; -import org.labkey.experiment.api.property.PropertyServiceImpl; -import org.labkey.experiment.api.property.RangeValidator; -import org.labkey.experiment.api.property.RegExValidator; -import org.labkey.experiment.api.property.StorageNameGenerator; -import org.labkey.experiment.api.property.StorageProvisionerImpl; -import org.labkey.experiment.api.property.TextChoiceValidator; -import org.labkey.experiment.controllers.exp.ExperimentController; -import org.labkey.experiment.controllers.property.PropertyController; -import org.labkey.experiment.defaults.DefaultValueServiceImpl; -import org.labkey.experiment.lineage.ExpLineageServiceImpl; -import org.labkey.experiment.lineage.LineagePerfTest; -import org.labkey.experiment.pipeline.ExperimentPipelineProvider; -import org.labkey.experiment.pipeline.XarTestPipelineJob; -import org.labkey.experiment.samples.DataClassFolderImporter; -import org.labkey.experiment.samples.DataClassFolderWriter; -import org.labkey.experiment.samples.SampleStatusFolderImporter; -import org.labkey.experiment.samples.SampleTimelineAuditProvider; -import org.labkey.experiment.samples.SampleTypeFolderImporter; -import org.labkey.experiment.samples.SampleTypeFolderWriter; -import org.labkey.experiment.security.DataClassDesignerRole; -import org.labkey.experiment.security.SampleTypeDesignerRole; -import org.labkey.experiment.types.TypesController; -import org.labkey.experiment.xar.FolderXarImporterFactory; -import org.labkey.experiment.xar.FolderXarWriterFactory; - -import java.util.ArrayList; -import java.util.Collection; -import java.util.Collections; -import java.util.HashMap; -import java.util.HashSet; -import java.util.LinkedHashSet; -import java.util.LinkedList; -import java.util.List; -import java.util.Map; -import java.util.Set; -import java.util.function.Supplier; -import java.util.stream.Collectors; - -import static org.labkey.api.data.ColumnRenderPropertiesImpl.STORAGE_UNIQUE_ID_CONCEPT_URI; -import static org.labkey.api.data.ColumnRenderPropertiesImpl.TEXT_CHOICE_CONCEPT_URI; -import static org.labkey.api.exp.api.ExperimentService.MODULE_NAME; - -public class ExperimentModule extends SpringModule -{ - private static final String SAMPLE_TYPE_WEB_PART_NAME = "Sample Types"; - private static final String PROTOCOL_WEB_PART_NAME = "Protocols"; - - public static final String AMOUNT_AND_UNIT_UPGRADE_PROP = "AmountAndUnitAudit"; - public static final String TRANSACTION_ID_PROP = "AuditTransactionId"; - public static final String AUDIT_COUNT_PROP = "AuditRecordCount"; - public static final String EXPERIMENT_RUN_WEB_PART_NAME = "Experiment Runs"; - - @Override - public String getName() - { - return MODULE_NAME; - } - - @Override - public Double getSchemaVersion() - { - return 26.001; - } - - @Nullable - @Override - public UpgradeCode getUpgradeCode() - { - return new ExperimentUpgradeCode(); - } - - @Override - protected void init() - { - addController("experiment", ExperimentController.class); - addController("experiment-types", TypesController.class); - addController("property", PropertyController.class); - ExperimentService.setInstance(new ExperimentServiceImpl()); - SampleTypeService.setInstance(new SampleTypeServiceImpl()); - DefaultValueService.setInstance(new DefaultValueServiceImpl()); - StorageProvisioner.setInstance(StorageProvisionerImpl.get()); - ExpLineageService.setInstance(new ExpLineageServiceImpl()); - - PropertyServiceImpl propertyServiceImpl = new PropertyServiceImpl(); - PropertyService.setInstance(propertyServiceImpl); - UsageMetricsService.get().registerUsageMetrics(getName(), propertyServiceImpl); - - UsageMetricsService.get().registerUsageMetrics(getName(), FileLinkMetricsProvider.getInstance()); - - ExperimentProperty.register(); - SamplesSchema.register(this); - ExpSchema.register(this); - - PropertyService.get().registerDomainKind(new SampleTypeDomainKind()); - PropertyService.get().registerDomainKind(new DataClassDomainKind()); - PropertyService.get().registerDomainKind(new VocabularyDomainKind()); - - QueryService.get().addCompareType(new ChildOfCompareType()); - QueryService.get().addCompareType(new ParentOfCompareType()); - QueryService.get().addCompareType(new LineageCompareType()); - QueryService.get().registerMethod(ChildOfMethod.NAME, new ChildOfMethod(), JdbcType.BOOLEAN, 2, 3); - QueryService.get().registerMethod(ParentOfMethod.NAME, new ParentOfMethod(), JdbcType.BOOLEAN, 2, 3); - QueryService.get().addQueryListener(new ExperimentQueryChangeListener()); - QueryService.get().addQueryListener(new PropertyQueryChangeListener()); - - PropertyService.get().registerValidatorKind(new RegExValidator()); - PropertyService.get().registerValidatorKind(new RangeValidator()); - PropertyService.get().registerValidatorKind(new LookupValidator()); - PropertyService.get().registerValidatorKind(new LengthValidator()); - PropertyService.get().registerValidatorKind(new TextChoiceValidator()); - - ExperimentService.get().registerExperimentDataHandler(new DefaultExperimentDataHandler()); - ExperimentService.get().registerProtocolInputCriteria(new FilterProtocolInputCriteria.Factory()); - ExperimentService.get().registerNameExpressionType("sampletype", "exp", "MaterialSource", "nameexpression"); - ExperimentService.get().registerNameExpressionType("aliquots", "exp", "MaterialSource", "aliquotnameexpression"); - ExperimentService.get().registerNameExpressionType("dataclass", "exp", "DataClass", "nameexpression"); - - OptionalFeatureService.get().addExperimentalFeatureFlag(AppProps.EXPERIMENTAL_RESOLVE_PROPERTY_URI_COLUMNS, "Resolve property URIs as columns on experiment tables", - "If a column is not found on an experiment table, attempt to resolve the column name as a Property URI and add it as a property column", false); - if (CoreSchema.getInstance().getSqlDialect().isSqlServer()) - { - OptionalFeatureService.get().addExperimentalFeatureFlag(NameGenerator.EXPERIMENTAL_WITH_COUNTER, "Use strict incremental withCounter and rootSampleCount expression", - "When withCounter or rootSampleCount is used in name expression, make sure the count increments one-by-one and does not jump.", true); - } - else - { - OptionalFeatureService.get().addExperimentalFeatureFlag(NameGenerator.EXPERIMENTAL_ALLOW_GAP_COUNTER, "Allow gap with withCounter and rootSampleCount expression", - "Check this option if gaps in the count generated by withCounter or rootSampleCount name expression are allowed.", true); - } - OptionalFeatureService.get().addExperimentalFeatureFlag(AppProps.QUANTITY_COLUMN_SUFFIX_TESTING, "Quantity column suffix testing", - "If a column name contains a \"__\" suffix, this feature allows for testing it as a Quantity display column", false); - OptionalFeatureService.get().addExperimentalFeatureFlag(ExperimentService.EXPERIMENTAL_FEATURE_FROM_EXPANCESTORS, "SQL syntax: 'FROM EXPANCESTORS()'", - "Support for querying lineage of experiment objects", false); - OptionalFeatureService.get().addExperimentalFeatureFlag(SampleTypeUpdateServiceDI.EXPERIMENTAL_FEATURE_ALLOW_ROW_ID_SAMPLE_MERGE, "Allow RowId to be accepted when merging samples", - "If the incoming data includes a RowId column we will allow the column but ignore it's values.", false); - - RoleManager.registerPermission(new DesignVocabularyPermission(), true); - RoleManager.registerRole(new SampleTypeDesignerRole()); - RoleManager.registerRole(new DataClassDesignerRole()); - - AttachmentService.get().registerAttachmentParentType(ExpRunAttachmentType.get()); - AttachmentService.get().registerAttachmentParentType(ExpProtocolAttachmentType.get()); - - WebdavService.get().addExpDataProvider((path, container) -> ExperimentService.get().getAllExpDataByURL(path, container)); - ExperimentService.get().registerObjectReferencer(ExperimentServiceImpl.get()); - - addModuleProperty(new LineageMaximumDepthModuleProperty(this)); - WarningService.get().register(new ExperimentWarningProvider()); - } - - @Override - public boolean hasScripts() - { - return true; - } - - @Override - @NotNull - protected Collection createWebPartFactories() - { - List result = new ArrayList<>(); - - BaseWebPartFactory runGroupsFactory = new BaseWebPartFactory(RunGroupWebPart.WEB_PART_NAME, WebPartFactory.LOCATION_BODY, WebPartFactory.LOCATION_RIGHT) - { - @Override - public WebPartView getWebPartView(@NotNull ViewContext portalCtx, @NotNull Portal.WebPart webPart) - { - return new RunGroupWebPart(portalCtx, WebPartFactory.LOCATION_RIGHT.equalsIgnoreCase(webPart.getLocation()), webPart); - } - }; - runGroupsFactory.addLegacyNames("Experiments", "Experiment", "Experiment Navigator", "Narrow Experiments"); - result.add(runGroupsFactory); - - BaseWebPartFactory runTypesFactory = new BaseWebPartFactory(RunTypeWebPart.WEB_PART_NAME, WebPartFactory.LOCATION_BODY, WebPartFactory.LOCATION_RIGHT) - { - @Override - public WebPartView getWebPartView(@NotNull ViewContext portalCtx, @NotNull Portal.WebPart webPart) - { - return new RunTypeWebPart(); - } - }; - result.add(runTypesFactory); - - result.add(new ExperimentRunWebPartFactory()); - BaseWebPartFactory sampleTypeFactory = new BaseWebPartFactory(SAMPLE_TYPE_WEB_PART_NAME, WebPartFactory.LOCATION_BODY, WebPartFactory.LOCATION_RIGHT) - { - @Override - public WebPartView getWebPartView(@NotNull ViewContext portalCtx, @NotNull Portal.WebPart webPart) - { - return new SampleTypeWebPart(WebPartFactory.LOCATION_RIGHT.equalsIgnoreCase(webPart.getLocation()), portalCtx); - } - }; - sampleTypeFactory.addLegacyNames("Narrow Sample Sets", "Sample Sets"); - result.add(sampleTypeFactory); - result.add(new AlwaysAvailableWebPartFactory("Samples Menu", false, false, WebPartFactory.LOCATION_MENUBAR) { - @Override - public WebPartView getWebPartView(@NotNull ViewContext portalCtx, @NotNull Portal.WebPart webPart) - { - WebPartView view = new JspView<>("/org/labkey/experiment/samplesAndAnalytes.jsp", webPart); - view.setTitle("Samples"); - return view; - } - }); - - result.add(new AlwaysAvailableWebPartFactory("Data Classes", false, false, WebPartFactory.LOCATION_BODY, WebPartFactory.LOCATION_RIGHT) { - @Override - public WebPartView getWebPartView(@NotNull ViewContext portalCtx, @NotNull Portal.WebPart webPart) - { - return new DataClassWebPart(WebPartFactory.LOCATION_RIGHT.equalsIgnoreCase(webPart.getLocation()), portalCtx, webPart); - } - }); - - BaseWebPartFactory narrowProtocolFactory = new BaseWebPartFactory(PROTOCOL_WEB_PART_NAME, WebPartFactory.LOCATION_RIGHT) - { - @Override - public WebPartView getWebPartView(@NotNull ViewContext portalCtx, @NotNull Portal.WebPart webPart) - { - return new ProtocolWebPart(WebPartFactory.LOCATION_RIGHT.equalsIgnoreCase(webPart.getLocation()), portalCtx); - } - }; - narrowProtocolFactory.addLegacyNames("Narrow Protocols"); - result.add(narrowProtocolFactory); - - return result; - } - - private void addDataResourceResolver(String categoryName) - { - SearchService.get().addResourceResolver(categoryName, new SearchService.ResourceResolver() - { - @Override - public WebdavResource resolve(@NotNull String resourceIdentifier) - { - ExpDataImpl data = ExpDataImpl.fromDocumentId(resourceIdentifier); - if (data == null) - return null; - - return data.createIndexDocument(null); - } - - @Override - public Map getCustomSearchJson(User user, @NotNull String resourceIdentifier) - { - ExpDataImpl data = ExpDataImpl.fromDocumentId(resourceIdentifier); - if (data == null) - return null; - - return ExperimentJSONConverter.serializeData(data, user, ExperimentJSONConverter.DEFAULT_SETTINGS).toMap(); - } - - @Override - public Map> getCustomSearchJsonMap(User user, @NotNull Collection resourceIdentifiers) - { - Map idDataMap = ExpDataImpl.fromDocumentIds(resourceIdentifiers); - if (idDataMap == null) - return null; - - Map> searchJsonMap = new HashMap<>(); - for (String resourceIdentifier : idDataMap.keySet()) - searchJsonMap.put(resourceIdentifier, ExperimentJSONConverter.serializeData(idDataMap.get(resourceIdentifier), user, ExperimentJSONConverter.DEFAULT_SETTINGS).toMap()); - return searchJsonMap; - } - }); - } - - private void addDataClassResourceResolver(String categoryName) - { - SearchService.get().addResourceResolver(categoryName, new SearchService.ResourceResolver(){ - @Override - public Map getCustomSearchJson(User user, @NotNull String resourceIdentifier) - { - int rowId = NumberUtils.toInt(resourceIdentifier.replace(categoryName + ":", "")); - if (rowId == 0) - return null; - - ExpDataClass dataClass = ExperimentService.get().getDataClass(rowId); - if (dataClass == null) - return null; - - Map properties = ExperimentJSONConverter.serializeExpObject(dataClass, null, ExperimentJSONConverter.DEFAULT_SETTINGS, user).toMap(); - - //Need to map to proper Icon - properties.put("type", "dataClass" + (dataClass.getCategory() != null ? ":" + dataClass.getCategory() : "")); - - return properties; - } - }); - } - - private void addSampleTypeResourceResolver(String categoryName) - { - SearchService.get().addResourceResolver(categoryName, new SearchService.ResourceResolver(){ - @Override - public Map getCustomSearchJson(User user, @NotNull String resourceIdentifier) - { - int rowId = NumberUtils.toInt(resourceIdentifier.replace(categoryName + ":", "")); - if (rowId == 0) - return null; - - ExpSampleType sampleType = SampleTypeService.get().getSampleType(rowId); - if (sampleType == null) - return null; - - Map properties = ExperimentJSONConverter.serializeExpObject(sampleType, null, ExperimentJSONConverter.DEFAULT_SETTINGS, user).toMap(); - - //Need to map to proper Icon - properties.put("type", "sampleSet"); - - return properties; - } - }); - } - - private void addSampleResourceResolver(String categoryName) - { - SearchService.get().addResourceResolver(categoryName, new SearchService.ResourceResolver(){ - @Override - public Map getCustomSearchJson(User user, @NotNull String resourceIdentifier) - { - int rowId = NumberUtils.toInt(resourceIdentifier.replace(categoryName + ":", "")); - if (rowId == 0) - return null; - - ExpMaterial material = ExperimentService.get().getExpMaterial(rowId); - if (material == null) - return null; - - return ExperimentJSONConverter.serializeMaterial(material, user, ExperimentJSONConverter.DEFAULT_SETTINGS).toMap(); - } - - @Override - public Map> getCustomSearchJsonMap(User user, @NotNull Collection resourceIdentifiers) - { - Set rowIds = new HashSet<>(); - Map rowIdIdentifierMap = new LongHashMap<>(); - for (String resourceIdentifier : resourceIdentifiers) - { - long rowId = NumberUtils.toLong(resourceIdentifier.replace(categoryName + ":", "")); - if (rowId != 0) - { - rowIds.add(rowId); - rowIdIdentifierMap.put(rowId, resourceIdentifier); - } - } - - Map> searchJsonMap = new HashMap<>(); - for (ExpMaterial material : ExperimentService.get().getExpMaterials(rowIds)) - { - searchJsonMap.put( - rowIdIdentifierMap.get(material.getRowId()), - ExperimentJSONConverter.serializeMaterial(material, user, ExperimentJSONConverter.DEFAULT_SETTINGS).toMap() - ); - } - - return searchJsonMap; - } - }); - } - - @Override - protected void startupAfterSpringConfig(ModuleContext moduleContext) - { - SearchService ss = SearchService.get(); -// ss.addSearchCategory(OntologyManager.conceptCategory); - ss.addSearchCategory(ExpSampleTypeImpl.searchCategory); - ss.addSearchCategory(ExpSampleTypeImpl.mediaSearchCategory); - ss.addSearchCategory(ExpMaterialImpl.searchCategory); - ss.addSearchCategory(ExpMaterialImpl.mediaSearchCategory); - ss.addSearchCategory(ExpDataClassImpl.SEARCH_CATEGORY); - ss.addSearchCategory(ExpDataClassImpl.MEDIA_SEARCH_CATEGORY); - ss.addSearchCategory(ExpDataImpl.expDataCategory); - ss.addSearchCategory(ExpDataImpl.expMediaDataCategory); - ss.addSearchResultTemplate(new ExpDataImpl.DataSearchResultTemplate()); - addDataResourceResolver(ExpDataImpl.expDataCategory.getName()); - addDataResourceResolver(ExpDataImpl.expMediaDataCategory.getName()); - addDataClassResourceResolver(ExpDataClassImpl.SEARCH_CATEGORY.getName()); - addDataClassResourceResolver(ExpDataClassImpl.MEDIA_SEARCH_CATEGORY.getName()); - addSampleTypeResourceResolver(ExpSampleTypeImpl.searchCategory.getName()); - addSampleTypeResourceResolver(ExpSampleTypeImpl.mediaSearchCategory.getName()); - addSampleResourceResolver(ExpMaterialImpl.searchCategory.getName()); - addSampleResourceResolver(ExpMaterialImpl.mediaSearchCategory.getName()); - ss.addDocumentProvider(ExperimentServiceImpl.get()); - - PipelineService.get().registerPipelineProvider(new ExperimentPipelineProvider(this)); - ExperimentService.get().registerExperimentRunTypeSource(container -> Collections.singleton(ExperimentRunType.ALL_RUNS_TYPE)); - ExperimentService.get().registerDataType(new LogDataType()); - - AuditLogService.get().registerAuditType(new DomainAuditProvider()); - AuditLogService.get().registerAuditType(new DomainPropertyAuditProvider()); - AuditLogService.get().registerAuditType(new ExperimentAuditProvider()); - AuditLogService.get().registerAuditType(new SampleTypeAuditProvider()); - AuditLogService.get().registerAuditType(new SampleTimelineAuditProvider()); - - FileContentService fileContentService = FileContentService.get(); - if (null != fileContentService) - { - fileContentService.addFileListener(new ExpDataFileListener()); - fileContentService.addFileListener(new TableUpdaterFileListener(ExperimentService.get().getTinfoExperimentRun(), "FilePathRoot", TableUpdaterFileListener.Type.fileRootPath, "RowId")); - fileContentService.addFileListener(new FileLinkFileListener()); - } - ContainerManager.addContainerListener(new ContainerManager.ContainerListener() - { - @Override - public void containerDeleted(Container c, User user) - { - try - { - ExperimentService.get().deleteAllExpObjInContainer(c, user); - } - catch (ExperimentException ee) - { - throw new RuntimeException(ee); - } - } - }, - // This is in the Last group because when a container is deleted, - // the Experiment listener needs to be called after the Study listener, - // because Study needs the metadata held by Experiment to delete properly. - // but it should be before the CoreContainerListener - ContainerManager.ContainerListener.Order.Last); - - if (ModuleLoader.getInstance().shouldInsertData()) - SystemProperty.registerProperties(); - - FolderSerializationRegistry folderRegistry = FolderSerializationRegistry.get(); - if (null != folderRegistry) - { - folderRegistry.addFactories(new FolderXarWriterFactory(), new FolderXarImporterFactory()); - folderRegistry.addWriterFactory(new SampleTypeFolderWriter.SampleTypeDesignWriter.Factory()); - folderRegistry.addWriterFactory(new SampleTypeFolderWriter.SampleTypeDataWriter.Factory()); - folderRegistry.addWriterFactory(new DataClassFolderWriter.DataClassDesignWriter.Factory()); - folderRegistry.addWriterFactory(new DataClassFolderWriter.DataClassDataWriter.Factory()); - folderRegistry.addImportFactory(new SampleTypeFolderImporter.Factory()); - folderRegistry.addImportFactory(new DataClassFolderImporter.Factory()); - folderRegistry.addImportFactory(new SampleStatusFolderImporter.Factory()); - } - - AttachmentService.get().registerAttachmentParentType(ExpDataClassType.get()); - - WebdavService.get().addProvider(new ScriptsResourceProvider()); - - SystemMaintenance.addTask(new FileLinkMetricsMaintenanceTask()); - - UsageMetricsService svc = UsageMetricsService.get(); - if (null != svc) - { - svc.registerUsageMetrics(getName(), () -> { - Map results = new HashMap<>(); - - DbSchema schema = ExperimentService.get().getSchema(); - if (AssayService.get() != null) - { - Map assayMetrics = new HashMap<>(); - SQLFragment baseRunSQL = new SQLFragment("SELECT COUNT(*) FROM ").append(ExperimentService.get().getTinfoExperimentRun(), "r").append(" WHERE lsid LIKE ?"); - SQLFragment baseProtocolSQL = new SQLFragment("SELECT * FROM ").append(ExperimentService.get().getTinfoProtocol(), "p").append(" WHERE lsid LIKE ? AND ApplicationType = ?"); - for (AssayProvider assayProvider : AssayService.get().getAssayProviders()) - { - Map protocolMetrics = new HashMap<>(); - - // Run count across all assay designs of this type - SQLFragment runSQL = new SQLFragment(baseRunSQL); - runSQL.add(Lsid.namespaceLikeString(assayProvider.getRunLSIDPrefix())); - protocolMetrics.put("runCount", new SqlSelector(schema, runSQL).getObject(Long.class)); - - // Number of assay designs of this type - SQLFragment protocolSQL = new SQLFragment(baseProtocolSQL); - protocolSQL.add(assayProvider.getProtocolPattern()); - protocolSQL.add(ExpProtocol.ApplicationType.ExperimentRun.toString()); - List protocols = new SqlSelector(schema, protocolSQL).getArrayList(Protocol.class); - protocolMetrics.put("protocolCount", protocols.size()); - - List wrappedProtocols = protocols.stream().map(ExpProtocolImpl::new).collect(Collectors.toList()); - - protocolMetrics.put("resultRowCount", assayProvider.getResultRowCount(wrappedProtocols)); - - // Primary implementation class - protocolMetrics.put("implementingClass", assayProvider.getClass()); - - assayMetrics.put(assayProvider.getName(), protocolMetrics); - } - assayMetrics.put("autoLinkedAssayCount", new SqlSelector(schema, "SELECT COUNT(*) FROM exp.protocol EP JOIN exp.objectPropertiesView OP ON EP.lsid = OP.objecturi WHERE OP.propertyuri = 'terms.labkey.org#AutoCopyTargetContainer'").getObject(Long.class)); - assayMetrics.put("protocolsWithTransformScriptCount", new SqlSelector(schema, "SELECT COUNT(*) FROM exp.protocol EP JOIN exp.objectPropertiesView OP ON EP.lsid = OP.objecturi WHERE OP.name = 'TransformScript' AND status = 'Active'").getObject(Long.class)); - assayMetrics.put("protocolsWithTransformScriptRunOnEditCount", new SqlSelector(schema, "SELECT COUNT(*) FROM exp.protocol EP JOIN exp.objectPropertiesView OP ON EP.lsid = OP.objecturi WHERE OP.name = 'TransformScript' AND status = 'Active' AND OP.stringvalue LIKE '%\"INSERT\"%'").getObject(Long.class)); - assayMetrics.put("protocolsWithTransformScriptRunOnImportCount", new SqlSelector(schema, "SELECT COUNT(*) FROM exp.protocol EP JOIN exp.objectPropertiesView OP ON EP.lsid = OP.objecturi WHERE OP.name = 'TransformScript' AND status = 'Active' AND OP.stringvalue LIKE '%\"INSERT\"%'").getObject(Long.class)); - - assayMetrics.put("standardAssayWithPlateSupportCount", new SqlSelector(schema, "SELECT COUNT(*) FROM exp.protocol EP JOIN exp.objectPropertiesView OP ON EP.lsid = OP.objecturi WHERE OP.name = 'PlateMetadata' AND floatValue = 1").getObject(Long.class)); - SQLFragment runsWithPlateSQL = new SQLFragment(""" - SELECT COUNT(*) FROM exp.experimentrun r - INNER JOIN exp.object o ON o.objectUri = r.lsid - INNER JOIN exp.objectproperty op ON op.objectId = o.objectId - WHERE op.propertyid IN ( - SELECT propertyid FROM exp.propertydescriptor WHERE name = ? AND lookupquery = ? - )"""); - assayMetrics.put("standardAssayRunsWithPlateTemplate", new SqlSelector(schema, new SQLFragment(runsWithPlateSQL).add("PlateTemplate").add("PlateTemplate")).getObject(Long.class)); - assayMetrics.put("standardAssayRunsWithPlateSet", new SqlSelector(schema, new SQLFragment(runsWithPlateSQL).add("PlateSet").add("PlateSet")).getObject(Long.class)); - - assayMetrics.put("assayRunsFileColumnCount", new SqlSelector(schema, """ - SELECT COUNT(DISTINCT DD.DomainURI) FROM - exp.PropertyDescriptor D\s - JOIN exp.PropertyDomain PD ON D.propertyId = PD.propertyid - JOIN exp.DomainDescriptor DD on PD.domainID = DD.domainId - WHERE DD.domainUri LIKE ? AND D.rangeURI = ?""", "urn:lsid:%:" + ExpProtocol.AssayDomainTypes.Run.getPrefix() + ".%", PropertyType.FILE_LINK.getTypeUri()).getObject(Long.class)); - - assayMetrics.put("assayResultsFileColumnCount", new SqlSelector(schema, """ - SELECT COUNT(DISTINCT DD.DomainURI) FROM - exp.PropertyDescriptor D\s - JOIN exp.PropertyDomain PD ON D.propertyId = PD.propertyid - JOIN exp.DomainDescriptor DD on PD.domainID = DD.domainId - WHERE DD.domainUri LIKE ? AND D.rangeURI = ?""", "urn:lsid:%:" + ExpProtocol.AssayDomainTypes.Result.getPrefix() + ".%", PropertyType.FILE_LINK.getTypeUri()).getObject(Long.class)); - - Map sampleLookupCountMetrics = new HashMap<>(); - SQLFragment baseAssaySampleLookupSQL = new SQLFragment("SELECT COUNT(*) FROM exp.propertydescriptor WHERE (lookupschema = 'samples' OR (lookupschema = 'exp' AND lookupquery = 'Materials')) AND propertyuri LIKE ?"); - - SQLFragment batchAssaySampleLookupSQL = new SQLFragment(baseAssaySampleLookupSQL); - batchAssaySampleLookupSQL.add("urn:lsid:%:" + ExpProtocol.AssayDomainTypes.Batch.getPrefix() + ".%"); - sampleLookupCountMetrics.put("batchDomain", new SqlSelector(schema, batchAssaySampleLookupSQL).getObject(Long.class)); - - SQLFragment runAssaySampleLookupSQL = new SQLFragment(baseAssaySampleLookupSQL); - runAssaySampleLookupSQL.add("urn:lsid:%:" + ExpProtocol.AssayDomainTypes.Run.getPrefix() + ".%"); - sampleLookupCountMetrics.put("runDomain", new SqlSelector(schema, runAssaySampleLookupSQL).getObject(Long.class)); - - SQLFragment resultAssaySampleLookupSQL = new SQLFragment(baseAssaySampleLookupSQL); - resultAssaySampleLookupSQL.add("urn:lsid:%:" + ExpProtocol.AssayDomainTypes.Result.getPrefix() + ".%"); - sampleLookupCountMetrics.put("resultDomain", new SqlSelector(schema, resultAssaySampleLookupSQL).getObject(Long.class)); - - SQLFragment resultAssayMultipleSampleLookupSQL = new SQLFragment( - """ - SELECT COUNT(*) FROM ( - SELECT PD.domainid, COUNT(*) AS PropCount - FROM exp.propertydescriptor D - JOIN exp.PropertyDomain PD ON D.propertyId = PD.propertyid - WHERE (lookupschema = 'samples' OR (lookupschema = 'exp' AND lookupquery = 'Materials')) - AND propertyuri LIKE ? - GROUP BY PD.domainid - ) X WHERE X.PropCount > 1""" - ); - resultAssayMultipleSampleLookupSQL.add("urn:lsid:%:" + ExpProtocol.AssayDomainTypes.Result.getPrefix() + ".%"); - sampleLookupCountMetrics.put("resultDomainWithMultiple", new SqlSelector(schema, resultAssayMultipleSampleLookupSQL).getObject(Long.class)); - - assayMetrics.put("sampleLookupCount", sampleLookupCountMetrics); - - - // Putting these metrics at the same level as the other BooleanColumnCount metrics (e.g., sampleTypeWithBooleanColumnCount) - results.put("assayResultWithBooleanColumnCount", new SqlSelector(schema, """ - SELECT COUNT(DISTINCT DD.DomainURI) FROM - exp.PropertyDescriptor D\s - JOIN exp.PropertyDomain PD ON D.propertyId = PD.propertyid - JOIN exp.DomainDescriptor DD on PD.domainID = DD.domainId - WHERE D.propertyURI LIKE ? AND D.rangeURI = ?""", "urn:lsid:%:" + ExpProtocol.AssayDomainTypes.Result.getPrefix() + ".%", PropertyType.BOOLEAN.getTypeUri()).getObject(Long.class)); - - results.put("assayRunWithBooleanColumnCount", new SqlSelector(schema, """ - SELECT COUNT(DISTINCT DD.DomainURI) FROM - exp.PropertyDescriptor D\s - JOIN exp.PropertyDomain PD ON D.propertyId = PD.propertyid - JOIN exp.DomainDescriptor DD on PD.domainID = DD.domainId - WHERE D.propertyURI LIKE ? AND D.rangeURI = ?""", "urn:lsid:%:" + ExpProtocol.AssayDomainTypes.Run.getPrefix() + ".%", PropertyType.BOOLEAN.getTypeUri()).getObject(Long.class)); - - results.put("assay", assayMetrics); - } - - results.put("autoLinkedSampleSetCount", new SqlSelector(schema, "SELECT COUNT(*) FROM exp.materialsource WHERE autoLinkTargetContainer IS NOT NULL").getObject(Long.class)); - results.put("sampleSetCount", new SqlSelector(schema, "SELECT COUNT(*) FROM exp.materialsource").getObject(Long.class)); - - if (schema.getSqlDialect().isPostgreSQL()) // SQLServer does not support regular expression queries - { - Collection> numSampleCounts = new SqlSelector(schema, """ - SELECT totalCount, numberNameCount FROM - (SELECT cpastype, COUNT(*) AS totalCount from exp.material GROUP BY cpastype) t - JOIN - (SELECT cpastype, COUNT(*) AS numberNameCount FROM exp.material m WHERE m.name SIMILAR TO '[0-9.]*' GROUP BY cpastype) ns - ON t.cpastype = ns.cpastype""").getMapCollection(); - results.put("sampleSetWithNumberNamesCount", numSampleCounts.size()); - results.put("sampleSetWithOnlyNumberNamesCount", numSampleCounts.stream().filter( - map -> (Long) map.get("totalCount") > 0 && map.get("totalCount") == map.get("numberNameCount") - ).count()); - } - UserSchema userSchema = AuditLogService.getAuditLogSchema(User.getSearchUser(), ContainerManager.getRoot()); - FilteredTable table = (FilteredTable) userSchema.getTable(SampleTimelineAuditEvent.EVENT_TYPE); - - SQLFragment sql = new SQLFragment("SELECT COUNT(*)\n" + - " FROM (\n" + - " -- updates that are marked as lineage updates\n" + - " (SELECT DISTINCT transactionId\n" + - " FROM " + table.getRealTable().getFromSQL("").getSQL() +"\n" + - " WHERE islineageupdate = " + schema.getSqlDialect().getBooleanTRUE() + "\n" + - " AND comment = 'Sample was updated.'\n" + - " ) a1\n" + - " JOIN\n" + - " -- but have associated entries that are not lineage updates\n" + - " (SELECT DISTINCT transactionid\n" + - " FROM " + table.getRealTable().getFromSQL("").getSQL() + "\n" + - " WHERE islineageupdate = " + schema.getSqlDialect().getBooleanFALSE() + ") a2\n" + - " ON a1.transactionid = a2.transactionid\n" + - " )"); - - results.put("sampleLineageAuditDiscrepancyCount", new SqlSelector(schema, sql.getSQL()).getObject(Long.class)); - - results.put("sampleCount", new SqlSelector(schema, "SELECT COUNT(*) FROM exp.material").getObject(Long.class)); - results.put("aliquotCount", new SqlSelector(schema, "SELECT COUNT(*) FROM exp.material where aliquotedfromlsid IS NOT NULL").getObject(Long.class)); - results.put("sampleNullAmountCount", new SqlSelector(schema, "SELECT COUNT(*) FROM exp.material WHERE storedamount IS NULL").getObject(Long.class)); - results.put("sampleNegativeAmountCount", new SqlSelector(schema, "SELECT COUNT(*) FROM exp.material WHERE storedamount < 0").getObject(Long.class)); - results.put("sampleUnitsDifferCount", new SqlSelector(schema, "SELECT COUNT(*) from exp.material m JOIN exp.materialSource s ON m.materialsourceid = s.rowid WHERE m.units != s.metricunit").getObject(Long.class)); - results.put("sampleTypesWithoutUnitsCount", new SqlSelector(schema, "SELECT COUNT(*) from exp.materialSource WHERE category IS NULL AND metricunit IS NULL").getObject(Long.class)); - results.put("sampleTypesWithMassTypeUnit", new SqlSelector(schema, "SELECT COUNT(*) from exp.materialSource WHERE category IS NULL AND metricunit IN ('kg', 'g', 'mg', 'ug', 'ng')").getObject(Long.class)); - results.put("sampleTypesWithVolumeTypeUnit", new SqlSelector(schema, "SELECT COUNT(*) from exp.materialSource WHERE category IS NULL AND metricunit IN ('L', 'mL', 'uL')").getObject(Long.class)); - results.put("sampleTypesWithCountTypeUnit", new SqlSelector(schema, "SELECT COUNT(*) from exp.materialSource WHERE category IS NULL AND metricunit = ?", "unit").getObject(Long.class)); - - results.put("duplicateSampleMaterialNameCount", new SqlSelector(schema, "SELECT COUNT(*) as duplicateCount FROM " + - "(SELECT name, cpastype FROM exp.material WHERE cpastype <> 'Material' GROUP BY name, cpastype HAVING COUNT(*) > 1) d").getObject(Long.class)); - results.put("duplicateSpecimenMaterialNameCount", new SqlSelector(schema, "SELECT COUNT(*) as duplicateCount FROM " + - "(SELECT name, cpastype FROM exp.material WHERE cpastype = 'Material' GROUP BY name, cpastype HAVING COUNT(*) > 1) d").getObject(Long.class)); - String duplicateCaseInsensitiveSampleNameCountSql = """ - SELECT COUNT(*) FROM - ( - SELECT 1 AS found - FROM exp.material - WHERE materialsourceid IS NOT NULL - GROUP BY LOWER(name), materialsourceid - HAVING COUNT(*) > 1 - ) AS duplicates - """; - String duplicateCaseInsensitiveDataNameCountSql = """ - SELECT COUNT(*) FROM - ( - SELECT 1 AS found - FROM exp.data - WHERE classid IS NOT NULL - GROUP BY LOWER(name), classid - HAVING COUNT(*) > 1 - ) AS duplicates - """; - results.put("duplicateCaseInsensitiveSampleNameCount", new SqlSelector(schema, duplicateCaseInsensitiveSampleNameCountSql).getObject(Long.class)); - results.put("duplicateCaseInsensitiveDataNameCount", new SqlSelector(schema, duplicateCaseInsensitiveDataNameCountSql).getObject(Long.class)); - - results.put("dataClassCount", new SqlSelector(schema, "SELECT COUNT(*) FROM exp.dataclass").getObject(Long.class)); - results.put("dataClassRowCount", new SqlSelector(schema, "SELECT COUNT(*) FROM exp.data WHERE classid IN (SELECT rowid FROM exp.dataclass)").getObject(Long.class)); - results.put("dataWithDataParentsCount", new SqlSelector(schema, "SELECT COUNT(DISTINCT d.sourceApplicationId) FROM exp.data d\n" + - "JOIN exp.datainput di ON di.targetapplicationid = d.sourceapplicationid").getObject(Long.class)); - if (schema.getSqlDialect().isPostgreSQL()) - { - Collection> numDataClassObjectsCounts = new SqlSelector(schema, """ - SELECT totalCount, numberNameCount FROM - (SELECT cpastype, COUNT(*) AS totalCount from exp.data GROUP BY cpastype) t - JOIN - (SELECT cpastype, COUNT(*) AS numberNameCount FROM exp.data m WHERE m.name SIMILAR TO '[0-9.]*' GROUP BY cpastype) ns - ON t.cpastype = ns.cpastype""").getMapCollection(); - results.put("dataClassWithNumberNamesCount", numDataClassObjectsCounts.size()); - results.put("dataClassWithOnlyNumberNamesCount", numDataClassObjectsCounts.stream().filter(map -> - (Long) map.get("totalCount") > 0 && map.get("totalCount") == map.get("numberNameCount")).count()); - } - - results.put("ontologyPrincipalConceptCodeCount", new SqlSelector(schema, "SELECT COUNT(*) FROM exp.propertydescriptor WHERE principalconceptcode IS NOT NULL").getObject(Long.class)); - results.put("ontologyLookupColumnCount", new SqlSelector(schema, "SELECT COUNT(*) FROM exp.propertydescriptor WHERE concepturi = ?", OntologyService.conceptCodeConceptURI).getObject(Long.class)); - results.put("ontologyConceptSubtreeCount", new SqlSelector(schema, "SELECT COUNT(*) FROM exp.propertydescriptor WHERE conceptsubtree IS NOT NULL").getObject(Long.class)); - results.put("ontologyConceptImportColumnCount", new SqlSelector(schema, "SELECT COUNT(*) FROM exp.propertydescriptor WHERE conceptimportcolumn IS NOT NULL").getObject(Long.class)); - results.put("ontologyConceptLabelColumnCount", new SqlSelector(schema, "SELECT COUNT(*) FROM exp.propertydescriptor WHERE conceptlabelcolumn IS NOT NULL").getObject(Long.class)); - - results.put("scannableColumnCount", new SqlSelector(schema, "SELECT COUNT(*) FROM exp.propertydescriptor WHERE scannable = ?", true).getObject(Long.class)); - results.put("uniqueIdColumnCount", new SqlSelector(schema, "SELECT COUNT(*) FROM exp.propertydescriptor WHERE concepturi = ?", STORAGE_UNIQUE_ID_CONCEPT_URI).getObject(Long.class)); - results.put("sampleTypeWithUniqueIdCount", new SqlSelector(schema, """ - SELECT COUNT(DISTINCT DD.DomainURI) FROM - exp.PropertyDescriptor D\s - JOIN exp.PropertyDomain PD ON D.propertyId = PD.propertyid - JOIN exp.DomainDescriptor DD on PD.domainID = DD.domainId - WHERE D.conceptURI = ?""", STORAGE_UNIQUE_ID_CONCEPT_URI).getObject(Long.class)); - - results.put("fileColumnCount", new SqlSelector(schema, "SELECT COUNT(*) FROM exp.propertydescriptor WHERE rangeURI = ?", PropertyType.FILE_LINK.getTypeUri()).getObject(Long.class)); - results.put("sampleTypeWithFileColumnCount", new SqlSelector(schema, """ - SELECT COUNT(DISTINCT DD.DomainURI) FROM - exp.PropertyDescriptor D\s - JOIN exp.PropertyDomain PD ON D.propertyId = PD.propertyid - JOIN exp.DomainDescriptor DD on PD.domainID = DD.domainId - WHERE DD.storageSchemaName = ? AND D.rangeURI = ?""", SampleTypeDomainKind.PROVISIONED_SCHEMA_NAME, PropertyType.FILE_LINK.getTypeUri()).getObject(Long.class)); - results.put("sampleTypeWithBooleanColumnCount", new SqlSelector(schema, """ - SELECT COUNT(DISTINCT DD.DomainURI) FROM - exp.PropertyDescriptor D\s - JOIN exp.PropertyDomain PD ON D.propertyId = PD.propertyid - JOIN exp.DomainDescriptor DD on PD.domainID = DD.domainId - WHERE DD.storageSchemaName = ? AND D.rangeURI = ?""", SampleTypeDomainKind.PROVISIONED_SCHEMA_NAME, PropertyType.BOOLEAN.getTypeUri()).getObject(Long.class)); - - results.put("sampleTypeAliquotSpecificField", new SqlSelector(schema, """ - SELECT COUNT(DISTINCT D.PropertyURI) FROM - exp.PropertyDescriptor D\s - JOIN exp.PropertyDomain PD ON D.propertyId = PD.propertyid - JOIN exp.DomainDescriptor DD on PD.domainID = DD.domainId - WHERE DD.storageSchemaName = ? AND D.derivationDataScope = ?""", SampleTypeDomainKind.PROVISIONED_SCHEMA_NAME, ExpSchema.DerivationDataScopeType.ChildOnly.name()).getObject(Long.class)); - results.put("sampleTypeParentOnlyField", new SqlSelector(schema, """ - SELECT COUNT(DISTINCT D.PropertyURI) FROM - exp.PropertyDescriptor D\s - JOIN exp.PropertyDomain PD ON D.propertyId = PD.propertyid - JOIN exp.DomainDescriptor DD on PD.domainID = DD.domainId - WHERE DD.storageSchemaName = ? AND (D.derivationDataScope = ? OR D.derivationDataScope IS NULL)""", SampleTypeDomainKind.PROVISIONED_SCHEMA_NAME, ExpSchema.DerivationDataScopeType.ParentOnly.name()).getObject(Long.class)); - results.put("sampleTypeParentAndAliquotField", new SqlSelector(schema, """ - SELECT COUNT(DISTINCT D.PropertyURI) FROM - exp.PropertyDescriptor D\s - JOIN exp.PropertyDomain PD ON D.propertyId = PD.propertyid - JOIN exp.DomainDescriptor DD on PD.domainID = DD.domainId - WHERE DD.storageSchemaName = ? AND D.derivationDataScope = ?""", SampleTypeDomainKind.PROVISIONED_SCHEMA_NAME, ExpSchema.DerivationDataScopeType.All.name()).getObject(Long.class)); - - results.put("attachmentColumnCount", new SqlSelector(schema, "SELECT COUNT(*) FROM exp.propertydescriptor WHERE rangeURI = ?", PropertyType.ATTACHMENT.getTypeUri()).getObject(Long.class)); - results.put("dataClassWithAttachmentColumnCount", new SqlSelector(schema, """ - SELECT COUNT(DISTINCT DD.DomainURI) FROM - exp.PropertyDescriptor D\s - JOIN exp.PropertyDomain PD ON D.propertyId = PD.propertyid - JOIN exp.DomainDescriptor DD on PD.domainID = DD.domainId - WHERE DD.storageSchemaName = ? AND D.rangeURI = ?""", DataClassDomainKind.PROVISIONED_SCHEMA_NAME, PropertyType.ATTACHMENT.getTypeUri()).getObject(Long.class)); - results.put("dataClassWithBooleanColumnCount", new SqlSelector(schema, """ - SELECT COUNT(DISTINCT DD.DomainURI) FROM - exp.PropertyDescriptor D\s - JOIN exp.PropertyDomain PD ON D.propertyId = PD.propertyid - JOIN exp.DomainDescriptor DD on PD.domainID = DD.domainId - WHERE DD.storageSchemaName = ? AND D.rangeURI = ?""", DataClassDomainKind.PROVISIONED_SCHEMA_NAME, PropertyType.BOOLEAN.getTypeUri()).getObject(Long.class)); - - results.put("textChoiceColumnCount", new SqlSelector(schema, "SELECT COUNT(*) FROM exp.propertydescriptor WHERE concepturi = ?", TEXT_CHOICE_CONCEPT_URI).getObject(Long.class)); - - results.put("domainsWithDateTimeColumnCount", new SqlSelector(schema, """ - SELECT COUNT(DISTINCT DD.DomainURI) FROM - exp.PropertyDescriptor D\s - JOIN exp.PropertyDomain PD ON D.propertyId = PD.propertyid - JOIN exp.DomainDescriptor DD on PD.domainID = DD.domainId - WHERE D.rangeURI = ?""", PropertyType.DATE_TIME.getTypeUri()).getObject(Long.class)); - - results.put("domainsWithDateColumnCount", new SqlSelector(schema, """ - SELECT COUNT(DISTINCT DD.DomainURI) FROM - exp.PropertyDescriptor D\s - JOIN exp.PropertyDomain PD ON D.propertyId = PD.propertyid - JOIN exp.DomainDescriptor DD on PD.domainID = DD.domainId - WHERE D.rangeURI = ?""", PropertyType.DATE.getTypeUri()).getObject(Long.class)); - - results.put("domainsWithTimeColumnCount", new SqlSelector(schema, """ - SELECT COUNT(DISTINCT DD.DomainURI) FROM - exp.PropertyDescriptor D\s - JOIN exp.PropertyDomain PD ON D.propertyId = PD.propertyid - JOIN exp.DomainDescriptor DD on PD.domainID = DD.domainId - WHERE D.rangeURI = ?""", PropertyType.TIME.getTypeUri()).getObject(Long.class)); - - results.put("maxObjectObjectId", new SqlSelector(schema, "SELECT MAX(ObjectId) FROM exp.Object").getObject(Long.class)); - results.put("maxMaterialRowId", new SqlSelector(schema, "SELECT MAX(RowId) FROM exp.Material").getObject(Long.class)); - - results.putAll(ExperimentService.get().getDomainMetrics()); - - return results; - }); - } - - ExperimentMigrationSchemaHandler handler = new ExperimentMigrationSchemaHandler(); - DatabaseMigrationService.get().registerSchemaHandler(handler); - DatabaseMigrationService.get().registerTableHandler(new MigrationTableHandler() - { - @Override - public TableInfo getTableInfo() - { - return DbSchema.get("premium", DbSchemaType.Bare).getTable("Exclusions"); - } - - @Override - public void adjustFilter(TableInfo sourceTable, SimpleFilter filter, Set containers) - { - // Include experiment runs that were copied - FilterClause includedClause = handler.getIncludedRowIdClause(sourceTable, FieldKey.fromParts("RunId")); - if (includedClause != null) - filter.addClause(includedClause); - } - }); - DatabaseMigrationService.get().registerTableHandler(new MigrationTableHandler() - { - @Override - public TableInfo getTableInfo() - { - return DbSchema.get("premium", DbSchemaType.Bare).getTable("ExclusionMaps"); - } - - @Override - public void adjustFilter(TableInfo sourceTable, SimpleFilter filter, Set containers) - { - // Include experiment runs that were copied - FilterClause includedClause = handler.getIncludedRowIdClause(sourceTable, FieldKey.fromParts("ExclusionId", "RunId")); - if (includedClause != null) - filter.addClause(includedClause); - } - }); - DatabaseMigrationService.get().registerTableHandler(new MigrationTableHandler() - { - @Override - public TableInfo getTableInfo() - { - return DbSchema.get("assayrequest", DbSchemaType.Bare).getTable("RequestRunsJunction"); - } - - @Override - public void adjustFilter(TableInfo sourceTable, SimpleFilter filter, Set containers) - { - // Include experiment runs that were copied - FilterClause includedClause = handler.getIncludedRowIdClause(sourceTable, FieldKey.fromParts("RunId")); - if (includedClause != null) - filter.addClause(includedClause); - } - }); - DatabaseMigrationService.get().registerSchemaHandler(new SampleTypeMigrationSchemaHandler()); - DataClassMigrationSchemaHandler dcHandler = new DataClassMigrationSchemaHandler(); - DatabaseMigrationService.get().registerSchemaHandler(dcHandler); - ExperimentDeleteService.setInstance(dcHandler); - } - - @Override - @NotNull - public Collection getSummary(Container c) - { - Collection list = new LinkedList<>(); - int runGroupCount = ExperimentService.get().getExperiments(c, null, false, true).size(); - if (runGroupCount > 0) - list.add(StringUtilsLabKey.pluralize(runGroupCount, "Run Group")); - - User user = HttpView.currentContext().getUser(); - - Set runTypes = ExperimentService.get().getExperimentRunTypes(c); - for (ExperimentRunType runType : runTypes) - { - if (runType == ExperimentRunType.ALL_RUNS_TYPE) - continue; - - long runCount = runType.getRunCount(user, c); - if (runCount > 0) - list.add(runCount + " runs of type " + runType.getDescription()); - } - - int dataClassCount = ExperimentService.get().getDataClasses(c, false).size(); - if (dataClassCount > 0) - list.add(dataClassCount + " Data Class" + (dataClassCount > 1 ? "es" : "")); - - int sampleTypeCount = SampleTypeService.get().getSampleTypes(c, false).size(); - if (sampleTypeCount > 0) - list.add(sampleTypeCount + " Sample Type" + (sampleTypeCount > 1 ? "s" : "")); - - return list; - } - - @Override - public @NotNull ArrayList getDetailedSummary(Container c, User user) - { - ArrayList summaries = new ArrayList<>(); - - // Assay types - long assayTypeCount = AssayService.get().getAssayProtocols(c).stream().filter(p -> p.getContainer().equals(c)).count(); - if (assayTypeCount > 0) - summaries.add(new Summary(assayTypeCount, "Assay Type")); - - // Run count - int runGroupCount = ExperimentService.get().getExperiments(c, user, false, true).size(); - if (runGroupCount > 0) - summaries.add(new Summary(runGroupCount, "Assay run")); - - // Number of Data Classes - List dataClasses = ExperimentService.get().getDataClasses(c, false); - int dataClassCount = dataClasses.size(); - if (dataClassCount > 0) - summaries.add(new Summary(dataClassCount, "Data Class")); - - ExpSchema expSchema = new ExpSchema(user, c); - - // Individual Data Class row counts - { - // The table-level container filter is set to ensure data class types are included - // that may not be defined in the target container but may have rows of data in the target container - TableInfo table = ExpSchema.TableType.DataClasses.createTable(expSchema, null, ContainerFilter.Type.CurrentPlusProjectAndShared.create(c, user)); - - // Issue 47919: The "DataCount" column is filtered to only count data in the target container - if (table instanceof ExpDataClassTableImpl tableImpl) - tableImpl.setDataCountContainerFilter(ContainerFilter.Type.Current.create(c, user)); - - Set columns = new LinkedHashSet<>(); - columns.add(ExpDataClassTable.Column.Name.name()); - columns.add(ExpDataClassTable.Column.DataCount.name()); - - Map results = new TableSelector(table, columns).getValueMap(String.class); - for (var entry : results.entrySet()) - { - long count = entry.getValue().longValue(); - if (count > 0) - summaries.add(new Summary(count, entry.getKey())); - } - } - - // Sample Types - int sampleTypeCount = SampleTypeService.get().getSampleTypes(c, false).size(); - if (sampleTypeCount > 0) - summaries.add(new Summary(sampleTypeCount, "Sample Type")); - - // Individual Sample Type row counts - { - // The table-level container filter is set to ensure data class types are included - // that may not be defined in the target container but may have rows of data in the target container - TableInfo table = ExpSchema.TableType.SampleSets.createTable(expSchema, null, ContainerFilter.Type.CurrentPlusProjectAndShared.create(c, user)); - - // Issue 51557: The "SampleCount" column is filtered to only count data in the target container - if (table instanceof ExpSampleTypeTableImpl tableImpl) - tableImpl.setSampleCountContainerFilter(ContainerFilter.Type.Current.create(c, user)); - - Set columns = new LinkedHashSet<>(); - columns.add(ExpSampleTypeTable.Column.Name.name()); - columns.add(ExpSampleTypeTable.Column.SampleCount.name()); - - Map results = new TableSelector(table, columns).getValueMap(String.class); - for (var entry : results.entrySet()) - { - long count = entry.getValue().longValue(); - if (count > 0) - { - String name = entry.getKey(); - Summary s = name.equals("MixtureBatches") - ? new Summary(count, "Batch") - : new Summary(count, name); - summaries.add(s); - } - } - } - - return summaries; - } - - @Override - public @NotNull Set> getIntegrationTests() - { - return Set.of( - DomainImpl.TestCase.class, - DomainPropertyImpl.TestCase.class, - ExpDataTableImpl.TestCase.class, - ExperimentServiceImpl.AuditDomainUriTest.class, - ExperimentServiceImpl.LineageQueryTestCase.class, - ExperimentServiceImpl.ParseInputOutputAliasTestCase.class, - ExperimentServiceImpl.TestCase.class, - ExperimentStressTest.class, - LineagePerfTest.class, - LineageTest.class, - OntologyManager.TestCase.class, - PropertyServiceImpl.TestCase.class, - SampleTypeServiceImpl.TestCase.class, - StorageNameGenerator.TestCase.class, - StorageProvisionerImpl.TestCase.class, - UniqueValueCounterTestCase.class, - XarTestPipelineJob.TestCase.class - ); - } - - @Override - public @NotNull Collection>> getIntegrationTestFactories() - { - List>> list = new ArrayList<>(super.getIntegrationTestFactories()); - list.add(new JspTestCase("/org/labkey/experiment/api/ExpDataClassDataTestCase.jsp")); - list.add(new JspTestCase("/org/labkey/experiment/api/ExpSampleTypeTestCase.jsp")); - return list; - } - - @Override - public @NotNull Set> getUnitTests() - { - return Set.of( - GraphAlgorithms.TestCase.class, - LSIDRelativizer.TestCase.class, - Lsid.TestCase.class, - LsidUtils.TestCase.class, - PropertyController.TestCase.class, - Quantity.TestCase.class, - Unit.TestCase.class - ); - } - - @Override - @NotNull - public Collection getSchemaNames() - { - return List.of( - ExpSchema.SCHEMA_NAME, - DataClassDomainKind.PROVISIONED_SCHEMA_NAME, - SampleTypeDomainKind.PROVISIONED_SCHEMA_NAME - ); - } - - @NotNull - @Override - public Collection getProvisionedSchemaNames() - { - return PageFlowUtil.set(DataClassDomainKind.PROVISIONED_SCHEMA_NAME, SampleTypeDomainKind.PROVISIONED_SCHEMA_NAME); - } -} +/* + * Copyright (c) 2008-2019 LabKey Corporation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.labkey.experiment; + +import org.apache.commons.lang3.math.NumberUtils; +import org.jetbrains.annotations.NotNull; +import org.jetbrains.annotations.Nullable; +import org.labkey.api.admin.FolderSerializationRegistry; +import org.labkey.api.assay.AssayProvider; +import org.labkey.api.assay.AssayService; +import org.labkey.api.attachments.AttachmentService; +import org.labkey.api.audit.AuditLogService; +import org.labkey.api.audit.SampleTimelineAuditEvent; +import org.labkey.api.collections.LongHashMap; +import org.labkey.api.data.Container; +import org.labkey.api.data.ContainerFilter; +import org.labkey.api.data.ContainerManager; +import org.labkey.api.data.CoreSchema; +import org.labkey.api.data.DbSchema; +import org.labkey.api.data.DbSchemaType; +import org.labkey.api.data.JdbcType; +import org.labkey.api.data.NameGenerator; +import org.labkey.api.data.SQLFragment; +import org.labkey.api.data.SimpleFilter; +import org.labkey.api.data.SimpleFilter.FilterClause; +import org.labkey.api.data.SqlSelector; +import org.labkey.api.data.TableInfo; +import org.labkey.api.data.TableSelector; +import org.labkey.api.data.UpgradeCode; +import org.labkey.api.defaults.DefaultValueService; +import org.labkey.api.exp.ExperimentException; +import org.labkey.api.exp.ExperimentRunType; +import org.labkey.api.exp.Lsid; +import org.labkey.api.exp.OntologyManager; +import org.labkey.api.exp.PropertyType; +import org.labkey.api.exp.api.DefaultExperimentDataHandler; +import org.labkey.api.exp.api.ExpData; +import org.labkey.api.exp.api.ExpDataClass; +import org.labkey.api.exp.api.ExpLineageService; +import org.labkey.api.exp.api.ExpMaterial; +import org.labkey.api.exp.api.ExpProtocol; +import org.labkey.api.exp.api.ExpProtocolAttachmentType; +import org.labkey.api.exp.api.ExpRunAttachmentType; +import org.labkey.api.exp.api.ExpSampleType; +import org.labkey.api.exp.api.ExperimentJSONConverter; +import org.labkey.api.exp.api.ExperimentService; +import org.labkey.api.exp.api.FilterProtocolInputCriteria; +import org.labkey.api.exp.api.SampleTypeDomainKind; +import org.labkey.api.exp.api.SampleTypeService; +import org.labkey.api.exp.api.StorageProvisioner; +import org.labkey.api.exp.property.DomainAuditProvider; +import org.labkey.api.exp.property.DomainPropertyAuditProvider; +import org.labkey.api.exp.property.ExperimentProperty; +import org.labkey.api.exp.property.PropertyService; +import org.labkey.api.exp.property.SystemProperty; +import org.labkey.api.exp.query.ExpDataClassTable; +import org.labkey.api.exp.query.ExpSampleTypeTable; +import org.labkey.api.exp.query.ExpSchema; +import org.labkey.api.exp.query.SamplesSchema; +import org.labkey.api.exp.xar.LSIDRelativizer; +import org.labkey.api.exp.xar.LsidUtils; +import org.labkey.api.files.FileContentService; +import org.labkey.api.files.TableUpdaterFileListener; +import org.labkey.api.migration.DatabaseMigrationService; +import org.labkey.api.migration.ExperimentDeleteService; +import org.labkey.api.migration.MigrationTableHandler; +import org.labkey.api.module.ModuleContext; +import org.labkey.api.module.ModuleLoader; +import org.labkey.api.module.SpringModule; +import org.labkey.api.module.Summary; +import org.labkey.api.ontology.OntologyService; +import org.labkey.api.ontology.Quantity; +import org.labkey.api.ontology.Unit; +import org.labkey.api.pipeline.PipelineService; +import org.labkey.api.query.FieldKey; +import org.labkey.api.query.FilteredTable; +import org.labkey.api.query.QueryService; +import org.labkey.api.query.UserSchema; +import org.labkey.api.search.SearchService; +import org.labkey.api.security.User; +import org.labkey.api.security.roles.RoleManager; +import org.labkey.api.settings.AppProps; +import org.labkey.api.settings.OptionalFeatureService; +import org.labkey.api.usageMetrics.UsageMetricsService; +import org.labkey.api.util.GUID; +import org.labkey.api.util.JspTestCase; +import org.labkey.api.util.PageFlowUtil; +import org.labkey.api.util.StringUtilsLabKey; +import org.labkey.api.util.SystemMaintenance; +import org.labkey.api.view.AlwaysAvailableWebPartFactory; +import org.labkey.api.view.BaseWebPartFactory; +import org.labkey.api.view.HttpView; +import org.labkey.api.view.JspView; +import org.labkey.api.view.Portal; +import org.labkey.api.view.ViewContext; +import org.labkey.api.view.WebPartFactory; +import org.labkey.api.view.WebPartView; +import org.labkey.api.view.template.WarningService; +import org.labkey.api.vocabulary.security.DesignVocabularyPermission; +import org.labkey.api.webdav.WebdavResource; +import org.labkey.api.webdav.WebdavService; +import org.labkey.experiment.api.DataClassDomainKind; +import org.labkey.experiment.api.ExpDataClassImpl; +import org.labkey.experiment.api.ExpDataClassTableImpl; +import org.labkey.experiment.api.ExpDataClassType; +import org.labkey.experiment.api.ExpDataImpl; +import org.labkey.experiment.api.ExpDataTableImpl; +import org.labkey.experiment.api.ExpMaterialImpl; +import org.labkey.experiment.api.ExpProtocolImpl; +import org.labkey.experiment.api.ExpSampleTypeImpl; +import org.labkey.experiment.api.ExpSampleTypeTableImpl; +import org.labkey.experiment.api.ExperimentServiceImpl; +import org.labkey.experiment.api.ExperimentStressTest; +import org.labkey.experiment.api.GraphAlgorithms; +import org.labkey.experiment.api.LineageTest; +import org.labkey.experiment.api.LogDataType; +import org.labkey.experiment.api.Protocol; +import org.labkey.experiment.api.SampleTypeServiceImpl; +import org.labkey.experiment.api.SampleTypeUpdateServiceDI; +import org.labkey.experiment.api.UniqueValueCounterTestCase; +import org.labkey.experiment.api.VocabularyDomainKind; +import org.labkey.experiment.api.data.ChildOfCompareType; +import org.labkey.experiment.api.data.ChildOfMethod; +import org.labkey.experiment.api.data.LineageCompareType; +import org.labkey.experiment.api.data.ParentOfCompareType; +import org.labkey.experiment.api.data.ParentOfMethod; +import org.labkey.experiment.api.property.DomainImpl; +import org.labkey.experiment.api.property.DomainPropertyImpl; +import org.labkey.experiment.api.property.LengthValidator; +import org.labkey.experiment.api.property.LookupValidator; +import org.labkey.experiment.api.property.PropertyServiceImpl; +import org.labkey.experiment.api.property.RangeValidator; +import org.labkey.experiment.api.property.RegExValidator; +import org.labkey.experiment.api.property.StorageNameGenerator; +import org.labkey.experiment.api.property.StorageProvisionerImpl; +import org.labkey.experiment.api.property.TextChoiceValidator; +import org.labkey.experiment.controllers.exp.ExperimentController; +import org.labkey.experiment.controllers.property.PropertyController; +import org.labkey.experiment.defaults.DefaultValueServiceImpl; +import org.labkey.experiment.lineage.ExpLineageServiceImpl; +import org.labkey.experiment.lineage.LineagePerfTest; +import org.labkey.experiment.pipeline.ExperimentPipelineProvider; +import org.labkey.experiment.pipeline.XarTestPipelineJob; +import org.labkey.experiment.samples.DataClassFolderImporter; +import org.labkey.experiment.samples.DataClassFolderWriter; +import org.labkey.experiment.samples.SampleStatusFolderImporter; +import org.labkey.experiment.samples.SampleTimelineAuditProvider; +import org.labkey.experiment.samples.SampleTypeFolderImporter; +import org.labkey.experiment.samples.SampleTypeFolderWriter; +import org.labkey.experiment.security.DataClassDesignerRole; +import org.labkey.experiment.security.SampleTypeDesignerRole; +import org.labkey.experiment.types.TypesController; +import org.labkey.experiment.xar.FolderXarImporterFactory; +import org.labkey.experiment.xar.FolderXarWriterFactory; + +import java.util.ArrayList; +import java.util.Collection; +import java.util.Collections; +import java.util.HashMap; +import java.util.HashSet; +import java.util.LinkedHashSet; +import java.util.LinkedList; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.function.Supplier; +import java.util.stream.Collectors; + +import static org.labkey.api.data.ColumnRenderPropertiesImpl.STORAGE_UNIQUE_ID_CONCEPT_URI; +import static org.labkey.api.data.ColumnRenderPropertiesImpl.TEXT_CHOICE_CONCEPT_URI; +import static org.labkey.api.exp.api.ExperimentService.MODULE_NAME; + +public class ExperimentModule extends SpringModule +{ + private static final String SAMPLE_TYPE_WEB_PART_NAME = "Sample Types"; + private static final String PROTOCOL_WEB_PART_NAME = "Protocols"; + + public static final String AMOUNT_AND_UNIT_UPGRADE_PROP = "AmountAndUnitAudit"; + public static final String TRANSACTION_ID_PROP = "AuditTransactionId"; + public static final String AUDIT_COUNT_PROP = "AuditRecordCount"; + public static final String EXPERIMENT_RUN_WEB_PART_NAME = "Experiment Runs"; + + @Override + public String getName() + { + return MODULE_NAME; + } + + @Override + public Double getSchemaVersion() + { + return 26.001; + } + + @Nullable + @Override + public UpgradeCode getUpgradeCode() + { + return new ExperimentUpgradeCode(); + } + + @Override + protected void init() + { + addController("experiment", ExperimentController.class); + addController("experiment-types", TypesController.class); + addController("property", PropertyController.class); + ExperimentService.setInstance(new ExperimentServiceImpl()); + SampleTypeService.setInstance(new SampleTypeServiceImpl()); + DefaultValueService.setInstance(new DefaultValueServiceImpl()); + StorageProvisioner.setInstance(StorageProvisionerImpl.get()); + ExpLineageService.setInstance(new ExpLineageServiceImpl()); + + PropertyServiceImpl propertyServiceImpl = new PropertyServiceImpl(); + PropertyService.setInstance(propertyServiceImpl); + UsageMetricsService.get().registerUsageMetrics(getName(), propertyServiceImpl); + + UsageMetricsService.get().registerUsageMetrics(getName(), FileLinkMetricsProvider.getInstance()); + + ExperimentProperty.register(); + SamplesSchema.register(this); + ExpSchema.register(this); + + PropertyService.get().registerDomainKind(new SampleTypeDomainKind()); + PropertyService.get().registerDomainKind(new DataClassDomainKind()); + PropertyService.get().registerDomainKind(new VocabularyDomainKind()); + + QueryService.get().addCompareType(new ChildOfCompareType()); + QueryService.get().addCompareType(new ParentOfCompareType()); + QueryService.get().addCompareType(new LineageCompareType()); + QueryService.get().registerMethod(ChildOfMethod.NAME, new ChildOfMethod(), JdbcType.BOOLEAN, 2, 3); + QueryService.get().registerMethod(ParentOfMethod.NAME, new ParentOfMethod(), JdbcType.BOOLEAN, 2, 3); + QueryService.get().addQueryListener(new ExperimentQueryChangeListener()); + QueryService.get().addQueryListener(new PropertyQueryChangeListener()); + + PropertyService.get().registerValidatorKind(new RegExValidator()); + PropertyService.get().registerValidatorKind(new RangeValidator()); + PropertyService.get().registerValidatorKind(new LookupValidator()); + PropertyService.get().registerValidatorKind(new LengthValidator()); + PropertyService.get().registerValidatorKind(new TextChoiceValidator()); + + ExperimentService.get().registerExperimentDataHandler(new DefaultExperimentDataHandler()); + ExperimentService.get().registerProtocolInputCriteria(new FilterProtocolInputCriteria.Factory()); + ExperimentService.get().registerNameExpressionType("sampletype", "exp", "MaterialSource", "nameexpression"); + ExperimentService.get().registerNameExpressionType("aliquots", "exp", "MaterialSource", "aliquotnameexpression"); + ExperimentService.get().registerNameExpressionType("dataclass", "exp", "DataClass", "nameexpression"); + + OptionalFeatureService.get().addExperimentalFeatureFlag(AppProps.EXPERIMENTAL_RESOLVE_PROPERTY_URI_COLUMNS, "Resolve property URIs as columns on experiment tables", + "If a column is not found on an experiment table, attempt to resolve the column name as a Property URI and add it as a property column", false); + if (CoreSchema.getInstance().getSqlDialect().isSqlServer()) + { + OptionalFeatureService.get().addExperimentalFeatureFlag(NameGenerator.EXPERIMENTAL_WITH_COUNTER, "Use strict incremental withCounter and rootSampleCount expression", + "When withCounter or rootSampleCount is used in name expression, make sure the count increments one-by-one and does not jump.", true); + } + else + { + OptionalFeatureService.get().addExperimentalFeatureFlag(NameGenerator.EXPERIMENTAL_ALLOW_GAP_COUNTER, "Allow gap with withCounter and rootSampleCount expression", + "Check this option if gaps in the count generated by withCounter or rootSampleCount name expression are allowed.", true); + OptionalFeatureService.get().addExperimentalFeatureFlag(AppProps.MULTI_VALUE_TEXT_CHOICE, "Allow multi-value Text Choice properties", + "Support selecting more than one values for text choice fields", false); + } + OptionalFeatureService.get().addExperimentalFeatureFlag(AppProps.QUANTITY_COLUMN_SUFFIX_TESTING, "Quantity column suffix testing", + "If a column name contains a \"__\" suffix, this feature allows for testing it as a Quantity display column", false); + OptionalFeatureService.get().addExperimentalFeatureFlag(ExperimentService.EXPERIMENTAL_FEATURE_FROM_EXPANCESTORS, "SQL syntax: 'FROM EXPANCESTORS()'", + "Support for querying lineage of experiment objects", false); + OptionalFeatureService.get().addExperimentalFeatureFlag(SampleTypeUpdateServiceDI.EXPERIMENTAL_FEATURE_ALLOW_ROW_ID_SAMPLE_MERGE, "Allow RowId to be accepted when merging samples", + "If the incoming data includes a RowId column we will allow the column but ignore it's values.", false); + + RoleManager.registerPermission(new DesignVocabularyPermission(), true); + RoleManager.registerRole(new SampleTypeDesignerRole()); + RoleManager.registerRole(new DataClassDesignerRole()); + + AttachmentService.get().registerAttachmentParentType(ExpRunAttachmentType.get()); + AttachmentService.get().registerAttachmentParentType(ExpProtocolAttachmentType.get()); + + WebdavService.get().addExpDataProvider((path, container) -> ExperimentService.get().getAllExpDataByURL(path, container)); + ExperimentService.get().registerObjectReferencer(ExperimentServiceImpl.get()); + + addModuleProperty(new LineageMaximumDepthModuleProperty(this)); + WarningService.get().register(new ExperimentWarningProvider()); + } + + @Override + public boolean hasScripts() + { + return true; + } + + @Override + @NotNull + protected Collection createWebPartFactories() + { + List result = new ArrayList<>(); + + BaseWebPartFactory runGroupsFactory = new BaseWebPartFactory(RunGroupWebPart.WEB_PART_NAME, WebPartFactory.LOCATION_BODY, WebPartFactory.LOCATION_RIGHT) + { + @Override + public WebPartView getWebPartView(@NotNull ViewContext portalCtx, @NotNull Portal.WebPart webPart) + { + return new RunGroupWebPart(portalCtx, WebPartFactory.LOCATION_RIGHT.equalsIgnoreCase(webPart.getLocation()), webPart); + } + }; + runGroupsFactory.addLegacyNames("Experiments", "Experiment", "Experiment Navigator", "Narrow Experiments"); + result.add(runGroupsFactory); + + BaseWebPartFactory runTypesFactory = new BaseWebPartFactory(RunTypeWebPart.WEB_PART_NAME, WebPartFactory.LOCATION_BODY, WebPartFactory.LOCATION_RIGHT) + { + @Override + public WebPartView getWebPartView(@NotNull ViewContext portalCtx, @NotNull Portal.WebPart webPart) + { + return new RunTypeWebPart(); + } + }; + result.add(runTypesFactory); + + result.add(new ExperimentRunWebPartFactory()); + BaseWebPartFactory sampleTypeFactory = new BaseWebPartFactory(SAMPLE_TYPE_WEB_PART_NAME, WebPartFactory.LOCATION_BODY, WebPartFactory.LOCATION_RIGHT) + { + @Override + public WebPartView getWebPartView(@NotNull ViewContext portalCtx, @NotNull Portal.WebPart webPart) + { + return new SampleTypeWebPart(WebPartFactory.LOCATION_RIGHT.equalsIgnoreCase(webPart.getLocation()), portalCtx); + } + }; + sampleTypeFactory.addLegacyNames("Narrow Sample Sets", "Sample Sets"); + result.add(sampleTypeFactory); + result.add(new AlwaysAvailableWebPartFactory("Samples Menu", false, false, WebPartFactory.LOCATION_MENUBAR) { + @Override + public WebPartView getWebPartView(@NotNull ViewContext portalCtx, @NotNull Portal.WebPart webPart) + { + WebPartView view = new JspView<>("/org/labkey/experiment/samplesAndAnalytes.jsp", webPart); + view.setTitle("Samples"); + return view; + } + }); + + result.add(new AlwaysAvailableWebPartFactory("Data Classes", false, false, WebPartFactory.LOCATION_BODY, WebPartFactory.LOCATION_RIGHT) { + @Override + public WebPartView getWebPartView(@NotNull ViewContext portalCtx, @NotNull Portal.WebPart webPart) + { + return new DataClassWebPart(WebPartFactory.LOCATION_RIGHT.equalsIgnoreCase(webPart.getLocation()), portalCtx, webPart); + } + }); + + BaseWebPartFactory narrowProtocolFactory = new BaseWebPartFactory(PROTOCOL_WEB_PART_NAME, WebPartFactory.LOCATION_RIGHT) + { + @Override + public WebPartView getWebPartView(@NotNull ViewContext portalCtx, @NotNull Portal.WebPart webPart) + { + return new ProtocolWebPart(WebPartFactory.LOCATION_RIGHT.equalsIgnoreCase(webPart.getLocation()), portalCtx); + } + }; + narrowProtocolFactory.addLegacyNames("Narrow Protocols"); + result.add(narrowProtocolFactory); + + return result; + } + + private void addDataResourceResolver(String categoryName) + { + SearchService.get().addResourceResolver(categoryName, new SearchService.ResourceResolver() + { + @Override + public WebdavResource resolve(@NotNull String resourceIdentifier) + { + ExpDataImpl data = ExpDataImpl.fromDocumentId(resourceIdentifier); + if (data == null) + return null; + + return data.createIndexDocument(null); + } + + @Override + public Map getCustomSearchJson(User user, @NotNull String resourceIdentifier) + { + ExpDataImpl data = ExpDataImpl.fromDocumentId(resourceIdentifier); + if (data == null) + return null; + + return ExperimentJSONConverter.serializeData(data, user, ExperimentJSONConverter.DEFAULT_SETTINGS).toMap(); + } + + @Override + public Map> getCustomSearchJsonMap(User user, @NotNull Collection resourceIdentifiers) + { + Map idDataMap = ExpDataImpl.fromDocumentIds(resourceIdentifiers); + if (idDataMap == null) + return null; + + Map> searchJsonMap = new HashMap<>(); + for (String resourceIdentifier : idDataMap.keySet()) + searchJsonMap.put(resourceIdentifier, ExperimentJSONConverter.serializeData(idDataMap.get(resourceIdentifier), user, ExperimentJSONConverter.DEFAULT_SETTINGS).toMap()); + return searchJsonMap; + } + }); + } + + private void addDataClassResourceResolver(String categoryName) + { + SearchService.get().addResourceResolver(categoryName, new SearchService.ResourceResolver(){ + @Override + public Map getCustomSearchJson(User user, @NotNull String resourceIdentifier) + { + int rowId = NumberUtils.toInt(resourceIdentifier.replace(categoryName + ":", "")); + if (rowId == 0) + return null; + + ExpDataClass dataClass = ExperimentService.get().getDataClass(rowId); + if (dataClass == null) + return null; + + Map properties = ExperimentJSONConverter.serializeExpObject(dataClass, null, ExperimentJSONConverter.DEFAULT_SETTINGS, user).toMap(); + + //Need to map to proper Icon + properties.put("type", "dataClass" + (dataClass.getCategory() != null ? ":" + dataClass.getCategory() : "")); + + return properties; + } + }); + } + + private void addSampleTypeResourceResolver(String categoryName) + { + SearchService.get().addResourceResolver(categoryName, new SearchService.ResourceResolver(){ + @Override + public Map getCustomSearchJson(User user, @NotNull String resourceIdentifier) + { + int rowId = NumberUtils.toInt(resourceIdentifier.replace(categoryName + ":", "")); + if (rowId == 0) + return null; + + ExpSampleType sampleType = SampleTypeService.get().getSampleType(rowId); + if (sampleType == null) + return null; + + Map properties = ExperimentJSONConverter.serializeExpObject(sampleType, null, ExperimentJSONConverter.DEFAULT_SETTINGS, user).toMap(); + + //Need to map to proper Icon + properties.put("type", "sampleSet"); + + return properties; + } + }); + } + + private void addSampleResourceResolver(String categoryName) + { + SearchService.get().addResourceResolver(categoryName, new SearchService.ResourceResolver(){ + @Override + public Map getCustomSearchJson(User user, @NotNull String resourceIdentifier) + { + int rowId = NumberUtils.toInt(resourceIdentifier.replace(categoryName + ":", "")); + if (rowId == 0) + return null; + + ExpMaterial material = ExperimentService.get().getExpMaterial(rowId); + if (material == null) + return null; + + return ExperimentJSONConverter.serializeMaterial(material, user, ExperimentJSONConverter.DEFAULT_SETTINGS).toMap(); + } + + @Override + public Map> getCustomSearchJsonMap(User user, @NotNull Collection resourceIdentifiers) + { + Set rowIds = new HashSet<>(); + Map rowIdIdentifierMap = new LongHashMap<>(); + for (String resourceIdentifier : resourceIdentifiers) + { + long rowId = NumberUtils.toLong(resourceIdentifier.replace(categoryName + ":", "")); + if (rowId != 0) + { + rowIds.add(rowId); + rowIdIdentifierMap.put(rowId, resourceIdentifier); + } + } + + Map> searchJsonMap = new HashMap<>(); + for (ExpMaterial material : ExperimentService.get().getExpMaterials(rowIds)) + { + searchJsonMap.put( + rowIdIdentifierMap.get(material.getRowId()), + ExperimentJSONConverter.serializeMaterial(material, user, ExperimentJSONConverter.DEFAULT_SETTINGS).toMap() + ); + } + + return searchJsonMap; + } + }); + } + + @Override + protected void startupAfterSpringConfig(ModuleContext moduleContext) + { + SearchService ss = SearchService.get(); +// ss.addSearchCategory(OntologyManager.conceptCategory); + ss.addSearchCategory(ExpSampleTypeImpl.searchCategory); + ss.addSearchCategory(ExpSampleTypeImpl.mediaSearchCategory); + ss.addSearchCategory(ExpMaterialImpl.searchCategory); + ss.addSearchCategory(ExpMaterialImpl.mediaSearchCategory); + ss.addSearchCategory(ExpDataClassImpl.SEARCH_CATEGORY); + ss.addSearchCategory(ExpDataClassImpl.MEDIA_SEARCH_CATEGORY); + ss.addSearchCategory(ExpDataImpl.expDataCategory); + ss.addSearchCategory(ExpDataImpl.expMediaDataCategory); + ss.addSearchResultTemplate(new ExpDataImpl.DataSearchResultTemplate()); + addDataResourceResolver(ExpDataImpl.expDataCategory.getName()); + addDataResourceResolver(ExpDataImpl.expMediaDataCategory.getName()); + addDataClassResourceResolver(ExpDataClassImpl.SEARCH_CATEGORY.getName()); + addDataClassResourceResolver(ExpDataClassImpl.MEDIA_SEARCH_CATEGORY.getName()); + addSampleTypeResourceResolver(ExpSampleTypeImpl.searchCategory.getName()); + addSampleTypeResourceResolver(ExpSampleTypeImpl.mediaSearchCategory.getName()); + addSampleResourceResolver(ExpMaterialImpl.searchCategory.getName()); + addSampleResourceResolver(ExpMaterialImpl.mediaSearchCategory.getName()); + ss.addDocumentProvider(ExperimentServiceImpl.get()); + + PipelineService.get().registerPipelineProvider(new ExperimentPipelineProvider(this)); + ExperimentService.get().registerExperimentRunTypeSource(container -> Collections.singleton(ExperimentRunType.ALL_RUNS_TYPE)); + ExperimentService.get().registerDataType(new LogDataType()); + + AuditLogService.get().registerAuditType(new DomainAuditProvider()); + AuditLogService.get().registerAuditType(new DomainPropertyAuditProvider()); + AuditLogService.get().registerAuditType(new ExperimentAuditProvider()); + AuditLogService.get().registerAuditType(new SampleTypeAuditProvider()); + AuditLogService.get().registerAuditType(new SampleTimelineAuditProvider()); + + FileContentService fileContentService = FileContentService.get(); + if (null != fileContentService) + { + fileContentService.addFileListener(new ExpDataFileListener()); + fileContentService.addFileListener(new TableUpdaterFileListener(ExperimentService.get().getTinfoExperimentRun(), "FilePathRoot", TableUpdaterFileListener.Type.fileRootPath, "RowId")); + fileContentService.addFileListener(new FileLinkFileListener()); + } + ContainerManager.addContainerListener(new ContainerManager.ContainerListener() + { + @Override + public void containerDeleted(Container c, User user) + { + try + { + ExperimentService.get().deleteAllExpObjInContainer(c, user); + } + catch (ExperimentException ee) + { + throw new RuntimeException(ee); + } + } + }, + // This is in the Last group because when a container is deleted, + // the Experiment listener needs to be called after the Study listener, + // because Study needs the metadata held by Experiment to delete properly. + // but it should be before the CoreContainerListener + ContainerManager.ContainerListener.Order.Last); + + if (ModuleLoader.getInstance().shouldInsertData()) + SystemProperty.registerProperties(); + + FolderSerializationRegistry folderRegistry = FolderSerializationRegistry.get(); + if (null != folderRegistry) + { + folderRegistry.addFactories(new FolderXarWriterFactory(), new FolderXarImporterFactory()); + folderRegistry.addWriterFactory(new SampleTypeFolderWriter.SampleTypeDesignWriter.Factory()); + folderRegistry.addWriterFactory(new SampleTypeFolderWriter.SampleTypeDataWriter.Factory()); + folderRegistry.addWriterFactory(new DataClassFolderWriter.DataClassDesignWriter.Factory()); + folderRegistry.addWriterFactory(new DataClassFolderWriter.DataClassDataWriter.Factory()); + folderRegistry.addImportFactory(new SampleTypeFolderImporter.Factory()); + folderRegistry.addImportFactory(new DataClassFolderImporter.Factory()); + folderRegistry.addImportFactory(new SampleStatusFolderImporter.Factory()); + } + + AttachmentService.get().registerAttachmentParentType(ExpDataClassType.get()); + + WebdavService.get().addProvider(new ScriptsResourceProvider()); + + SystemMaintenance.addTask(new FileLinkMetricsMaintenanceTask()); + + UsageMetricsService svc = UsageMetricsService.get(); + if (null != svc) + { + svc.registerUsageMetrics(getName(), () -> { + Map results = new HashMap<>(); + + DbSchema schema = ExperimentService.get().getSchema(); + if (AssayService.get() != null) + { + Map assayMetrics = new HashMap<>(); + SQLFragment baseRunSQL = new SQLFragment("SELECT COUNT(*) FROM ").append(ExperimentService.get().getTinfoExperimentRun(), "r").append(" WHERE lsid LIKE ?"); + SQLFragment baseProtocolSQL = new SQLFragment("SELECT * FROM ").append(ExperimentService.get().getTinfoProtocol(), "p").append(" WHERE lsid LIKE ? AND ApplicationType = ?"); + for (AssayProvider assayProvider : AssayService.get().getAssayProviders()) + { + Map protocolMetrics = new HashMap<>(); + + // Run count across all assay designs of this type + SQLFragment runSQL = new SQLFragment(baseRunSQL); + runSQL.add(Lsid.namespaceLikeString(assayProvider.getRunLSIDPrefix())); + protocolMetrics.put("runCount", new SqlSelector(schema, runSQL).getObject(Long.class)); + + // Number of assay designs of this type + SQLFragment protocolSQL = new SQLFragment(baseProtocolSQL); + protocolSQL.add(assayProvider.getProtocolPattern()); + protocolSQL.add(ExpProtocol.ApplicationType.ExperimentRun.toString()); + List protocols = new SqlSelector(schema, protocolSQL).getArrayList(Protocol.class); + protocolMetrics.put("protocolCount", protocols.size()); + + List wrappedProtocols = protocols.stream().map(ExpProtocolImpl::new).collect(Collectors.toList()); + + protocolMetrics.put("resultRowCount", assayProvider.getResultRowCount(wrappedProtocols)); + + // Primary implementation class + protocolMetrics.put("implementingClass", assayProvider.getClass()); + + assayMetrics.put(assayProvider.getName(), protocolMetrics); + } + assayMetrics.put("autoLinkedAssayCount", new SqlSelector(schema, "SELECT COUNT(*) FROM exp.protocol EP JOIN exp.objectPropertiesView OP ON EP.lsid = OP.objecturi WHERE OP.propertyuri = 'terms.labkey.org#AutoCopyTargetContainer'").getObject(Long.class)); + assayMetrics.put("protocolsWithTransformScriptCount", new SqlSelector(schema, "SELECT COUNT(*) FROM exp.protocol EP JOIN exp.objectPropertiesView OP ON EP.lsid = OP.objecturi WHERE OP.name = 'TransformScript' AND status = 'Active'").getObject(Long.class)); + assayMetrics.put("protocolsWithTransformScriptRunOnEditCount", new SqlSelector(schema, "SELECT COUNT(*) FROM exp.protocol EP JOIN exp.objectPropertiesView OP ON EP.lsid = OP.objecturi WHERE OP.name = 'TransformScript' AND status = 'Active' AND OP.stringvalue LIKE '%\"INSERT\"%'").getObject(Long.class)); + assayMetrics.put("protocolsWithTransformScriptRunOnImportCount", new SqlSelector(schema, "SELECT COUNT(*) FROM exp.protocol EP JOIN exp.objectPropertiesView OP ON EP.lsid = OP.objecturi WHERE OP.name = 'TransformScript' AND status = 'Active' AND OP.stringvalue LIKE '%\"INSERT\"%'").getObject(Long.class)); + + assayMetrics.put("standardAssayWithPlateSupportCount", new SqlSelector(schema, "SELECT COUNT(*) FROM exp.protocol EP JOIN exp.objectPropertiesView OP ON EP.lsid = OP.objecturi WHERE OP.name = 'PlateMetadata' AND floatValue = 1").getObject(Long.class)); + SQLFragment runsWithPlateSQL = new SQLFragment(""" + SELECT COUNT(*) FROM exp.experimentrun r + INNER JOIN exp.object o ON o.objectUri = r.lsid + INNER JOIN exp.objectproperty op ON op.objectId = o.objectId + WHERE op.propertyid IN ( + SELECT propertyid FROM exp.propertydescriptor WHERE name = ? AND lookupquery = ? + )"""); + assayMetrics.put("standardAssayRunsWithPlateTemplate", new SqlSelector(schema, new SQLFragment(runsWithPlateSQL).add("PlateTemplate").add("PlateTemplate")).getObject(Long.class)); + assayMetrics.put("standardAssayRunsWithPlateSet", new SqlSelector(schema, new SQLFragment(runsWithPlateSQL).add("PlateSet").add("PlateSet")).getObject(Long.class)); + + assayMetrics.put("assayRunsFileColumnCount", new SqlSelector(schema, """ + SELECT COUNT(DISTINCT DD.DomainURI) FROM + exp.PropertyDescriptor D\s + JOIN exp.PropertyDomain PD ON D.propertyId = PD.propertyid + JOIN exp.DomainDescriptor DD on PD.domainID = DD.domainId + WHERE DD.domainUri LIKE ? AND D.rangeURI = ?""", "urn:lsid:%:" + ExpProtocol.AssayDomainTypes.Run.getPrefix() + ".%", PropertyType.FILE_LINK.getTypeUri()).getObject(Long.class)); + + assayMetrics.put("assayResultsFileColumnCount", new SqlSelector(schema, """ + SELECT COUNT(DISTINCT DD.DomainURI) FROM + exp.PropertyDescriptor D\s + JOIN exp.PropertyDomain PD ON D.propertyId = PD.propertyid + JOIN exp.DomainDescriptor DD on PD.domainID = DD.domainId + WHERE DD.domainUri LIKE ? AND D.rangeURI = ?""", "urn:lsid:%:" + ExpProtocol.AssayDomainTypes.Result.getPrefix() + ".%", PropertyType.FILE_LINK.getTypeUri()).getObject(Long.class)); + + Map sampleLookupCountMetrics = new HashMap<>(); + SQLFragment baseAssaySampleLookupSQL = new SQLFragment("SELECT COUNT(*) FROM exp.propertydescriptor WHERE (lookupschema = 'samples' OR (lookupschema = 'exp' AND lookupquery = 'Materials')) AND propertyuri LIKE ?"); + + SQLFragment batchAssaySampleLookupSQL = new SQLFragment(baseAssaySampleLookupSQL); + batchAssaySampleLookupSQL.add("urn:lsid:%:" + ExpProtocol.AssayDomainTypes.Batch.getPrefix() + ".%"); + sampleLookupCountMetrics.put("batchDomain", new SqlSelector(schema, batchAssaySampleLookupSQL).getObject(Long.class)); + + SQLFragment runAssaySampleLookupSQL = new SQLFragment(baseAssaySampleLookupSQL); + runAssaySampleLookupSQL.add("urn:lsid:%:" + ExpProtocol.AssayDomainTypes.Run.getPrefix() + ".%"); + sampleLookupCountMetrics.put("runDomain", new SqlSelector(schema, runAssaySampleLookupSQL).getObject(Long.class)); + + SQLFragment resultAssaySampleLookupSQL = new SQLFragment(baseAssaySampleLookupSQL); + resultAssaySampleLookupSQL.add("urn:lsid:%:" + ExpProtocol.AssayDomainTypes.Result.getPrefix() + ".%"); + sampleLookupCountMetrics.put("resultDomain", new SqlSelector(schema, resultAssaySampleLookupSQL).getObject(Long.class)); + + SQLFragment resultAssayMultipleSampleLookupSQL = new SQLFragment( + """ + SELECT COUNT(*) FROM ( + SELECT PD.domainid, COUNT(*) AS PropCount + FROM exp.propertydescriptor D + JOIN exp.PropertyDomain PD ON D.propertyId = PD.propertyid + WHERE (lookupschema = 'samples' OR (lookupschema = 'exp' AND lookupquery = 'Materials')) + AND propertyuri LIKE ? + GROUP BY PD.domainid + ) X WHERE X.PropCount > 1""" + ); + resultAssayMultipleSampleLookupSQL.add("urn:lsid:%:" + ExpProtocol.AssayDomainTypes.Result.getPrefix() + ".%"); + sampleLookupCountMetrics.put("resultDomainWithMultiple", new SqlSelector(schema, resultAssayMultipleSampleLookupSQL).getObject(Long.class)); + + assayMetrics.put("sampleLookupCount", sampleLookupCountMetrics); + + + // Putting these metrics at the same level as the other BooleanColumnCount metrics (e.g., sampleTypeWithBooleanColumnCount) + results.put("assayResultWithBooleanColumnCount", new SqlSelector(schema, """ + SELECT COUNT(DISTINCT DD.DomainURI) FROM + exp.PropertyDescriptor D\s + JOIN exp.PropertyDomain PD ON D.propertyId = PD.propertyid + JOIN exp.DomainDescriptor DD on PD.domainID = DD.domainId + WHERE D.propertyURI LIKE ? AND D.rangeURI = ?""", "urn:lsid:%:" + ExpProtocol.AssayDomainTypes.Result.getPrefix() + ".%", PropertyType.BOOLEAN.getTypeUri()).getObject(Long.class)); + + results.put("assayRunWithBooleanColumnCount", new SqlSelector(schema, """ + SELECT COUNT(DISTINCT DD.DomainURI) FROM + exp.PropertyDescriptor D\s + JOIN exp.PropertyDomain PD ON D.propertyId = PD.propertyid + JOIN exp.DomainDescriptor DD on PD.domainID = DD.domainId + WHERE D.propertyURI LIKE ? AND D.rangeURI = ?""", "urn:lsid:%:" + ExpProtocol.AssayDomainTypes.Run.getPrefix() + ".%", PropertyType.BOOLEAN.getTypeUri()).getObject(Long.class)); + + results.put("assay", assayMetrics); + } + + results.put("autoLinkedSampleSetCount", new SqlSelector(schema, "SELECT COUNT(*) FROM exp.materialsource WHERE autoLinkTargetContainer IS NOT NULL").getObject(Long.class)); + results.put("sampleSetCount", new SqlSelector(schema, "SELECT COUNT(*) FROM exp.materialsource").getObject(Long.class)); + + if (schema.getSqlDialect().isPostgreSQL()) // SQLServer does not support regular expression queries + { + Collection> numSampleCounts = new SqlSelector(schema, """ + SELECT totalCount, numberNameCount FROM + (SELECT cpastype, COUNT(*) AS totalCount from exp.material GROUP BY cpastype) t + JOIN + (SELECT cpastype, COUNT(*) AS numberNameCount FROM exp.material m WHERE m.name SIMILAR TO '[0-9.]*' GROUP BY cpastype) ns + ON t.cpastype = ns.cpastype""").getMapCollection(); + results.put("sampleSetWithNumberNamesCount", numSampleCounts.size()); + results.put("sampleSetWithOnlyNumberNamesCount", numSampleCounts.stream().filter( + map -> (Long) map.get("totalCount") > 0 && map.get("totalCount") == map.get("numberNameCount") + ).count()); + } + UserSchema userSchema = AuditLogService.getAuditLogSchema(User.getSearchUser(), ContainerManager.getRoot()); + FilteredTable table = (FilteredTable) userSchema.getTable(SampleTimelineAuditEvent.EVENT_TYPE); + + SQLFragment sql = new SQLFragment("SELECT COUNT(*)\n" + + " FROM (\n" + + " -- updates that are marked as lineage updates\n" + + " (SELECT DISTINCT transactionId\n" + + " FROM " + table.getRealTable().getFromSQL("").getSQL() +"\n" + + " WHERE islineageupdate = " + schema.getSqlDialect().getBooleanTRUE() + "\n" + + " AND comment = 'Sample was updated.'\n" + + " ) a1\n" + + " JOIN\n" + + " -- but have associated entries that are not lineage updates\n" + + " (SELECT DISTINCT transactionid\n" + + " FROM " + table.getRealTable().getFromSQL("").getSQL() + "\n" + + " WHERE islineageupdate = " + schema.getSqlDialect().getBooleanFALSE() + ") a2\n" + + " ON a1.transactionid = a2.transactionid\n" + + " )"); + + results.put("sampleLineageAuditDiscrepancyCount", new SqlSelector(schema, sql.getSQL()).getObject(Long.class)); + + results.put("sampleCount", new SqlSelector(schema, "SELECT COUNT(*) FROM exp.material").getObject(Long.class)); + results.put("aliquotCount", new SqlSelector(schema, "SELECT COUNT(*) FROM exp.material where aliquotedfromlsid IS NOT NULL").getObject(Long.class)); + results.put("sampleNullAmountCount", new SqlSelector(schema, "SELECT COUNT(*) FROM exp.material WHERE storedamount IS NULL").getObject(Long.class)); + results.put("sampleNegativeAmountCount", new SqlSelector(schema, "SELECT COUNT(*) FROM exp.material WHERE storedamount < 0").getObject(Long.class)); + results.put("sampleUnitsDifferCount", new SqlSelector(schema, "SELECT COUNT(*) from exp.material m JOIN exp.materialSource s ON m.materialsourceid = s.rowid WHERE m.units != s.metricunit").getObject(Long.class)); + results.put("sampleTypesWithoutUnitsCount", new SqlSelector(schema, "SELECT COUNT(*) from exp.materialSource WHERE category IS NULL AND metricunit IS NULL").getObject(Long.class)); + results.put("sampleTypesWithMassTypeUnit", new SqlSelector(schema, "SELECT COUNT(*) from exp.materialSource WHERE category IS NULL AND metricunit IN ('kg', 'g', 'mg', 'ug', 'ng')").getObject(Long.class)); + results.put("sampleTypesWithVolumeTypeUnit", new SqlSelector(schema, "SELECT COUNT(*) from exp.materialSource WHERE category IS NULL AND metricunit IN ('L', 'mL', 'uL')").getObject(Long.class)); + results.put("sampleTypesWithCountTypeUnit", new SqlSelector(schema, "SELECT COUNT(*) from exp.materialSource WHERE category IS NULL AND metricunit = ?", "unit").getObject(Long.class)); + + results.put("duplicateSampleMaterialNameCount", new SqlSelector(schema, "SELECT COUNT(*) as duplicateCount FROM " + + "(SELECT name, cpastype FROM exp.material WHERE cpastype <> 'Material' GROUP BY name, cpastype HAVING COUNT(*) > 1) d").getObject(Long.class)); + results.put("duplicateSpecimenMaterialNameCount", new SqlSelector(schema, "SELECT COUNT(*) as duplicateCount FROM " + + "(SELECT name, cpastype FROM exp.material WHERE cpastype = 'Material' GROUP BY name, cpastype HAVING COUNT(*) > 1) d").getObject(Long.class)); + String duplicateCaseInsensitiveSampleNameCountSql = """ + SELECT COUNT(*) FROM + ( + SELECT 1 AS found + FROM exp.material + WHERE materialsourceid IS NOT NULL + GROUP BY LOWER(name), materialsourceid + HAVING COUNT(*) > 1 + ) AS duplicates + """; + String duplicateCaseInsensitiveDataNameCountSql = """ + SELECT COUNT(*) FROM + ( + SELECT 1 AS found + FROM exp.data + WHERE classid IS NOT NULL + GROUP BY LOWER(name), classid + HAVING COUNT(*) > 1 + ) AS duplicates + """; + results.put("duplicateCaseInsensitiveSampleNameCount", new SqlSelector(schema, duplicateCaseInsensitiveSampleNameCountSql).getObject(Long.class)); + results.put("duplicateCaseInsensitiveDataNameCount", new SqlSelector(schema, duplicateCaseInsensitiveDataNameCountSql).getObject(Long.class)); + + results.put("dataClassCount", new SqlSelector(schema, "SELECT COUNT(*) FROM exp.dataclass").getObject(Long.class)); + results.put("dataClassRowCount", new SqlSelector(schema, "SELECT COUNT(*) FROM exp.data WHERE classid IN (SELECT rowid FROM exp.dataclass)").getObject(Long.class)); + results.put("dataWithDataParentsCount", new SqlSelector(schema, "SELECT COUNT(DISTINCT d.sourceApplicationId) FROM exp.data d\n" + + "JOIN exp.datainput di ON di.targetapplicationid = d.sourceapplicationid").getObject(Long.class)); + if (schema.getSqlDialect().isPostgreSQL()) + { + Collection> numDataClassObjectsCounts = new SqlSelector(schema, """ + SELECT totalCount, numberNameCount FROM + (SELECT cpastype, COUNT(*) AS totalCount from exp.data GROUP BY cpastype) t + JOIN + (SELECT cpastype, COUNT(*) AS numberNameCount FROM exp.data m WHERE m.name SIMILAR TO '[0-9.]*' GROUP BY cpastype) ns + ON t.cpastype = ns.cpastype""").getMapCollection(); + results.put("dataClassWithNumberNamesCount", numDataClassObjectsCounts.size()); + results.put("dataClassWithOnlyNumberNamesCount", numDataClassObjectsCounts.stream().filter(map -> + (Long) map.get("totalCount") > 0 && map.get("totalCount") == map.get("numberNameCount")).count()); + } + + results.put("ontologyPrincipalConceptCodeCount", new SqlSelector(schema, "SELECT COUNT(*) FROM exp.propertydescriptor WHERE principalconceptcode IS NOT NULL").getObject(Long.class)); + results.put("ontologyLookupColumnCount", new SqlSelector(schema, "SELECT COUNT(*) FROM exp.propertydescriptor WHERE concepturi = ?", OntologyService.conceptCodeConceptURI).getObject(Long.class)); + results.put("ontologyConceptSubtreeCount", new SqlSelector(schema, "SELECT COUNT(*) FROM exp.propertydescriptor WHERE conceptsubtree IS NOT NULL").getObject(Long.class)); + results.put("ontologyConceptImportColumnCount", new SqlSelector(schema, "SELECT COUNT(*) FROM exp.propertydescriptor WHERE conceptimportcolumn IS NOT NULL").getObject(Long.class)); + results.put("ontologyConceptLabelColumnCount", new SqlSelector(schema, "SELECT COUNT(*) FROM exp.propertydescriptor WHERE conceptlabelcolumn IS NOT NULL").getObject(Long.class)); + + results.put("scannableColumnCount", new SqlSelector(schema, "SELECT COUNT(*) FROM exp.propertydescriptor WHERE scannable = ?", true).getObject(Long.class)); + results.put("uniqueIdColumnCount", new SqlSelector(schema, "SELECT COUNT(*) FROM exp.propertydescriptor WHERE concepturi = ?", STORAGE_UNIQUE_ID_CONCEPT_URI).getObject(Long.class)); + results.put("sampleTypeWithUniqueIdCount", new SqlSelector(schema, """ + SELECT COUNT(DISTINCT DD.DomainURI) FROM + exp.PropertyDescriptor D\s + JOIN exp.PropertyDomain PD ON D.propertyId = PD.propertyid + JOIN exp.DomainDescriptor DD on PD.domainID = DD.domainId + WHERE D.conceptURI = ?""", STORAGE_UNIQUE_ID_CONCEPT_URI).getObject(Long.class)); + + results.put("fileColumnCount", new SqlSelector(schema, "SELECT COUNT(*) FROM exp.propertydescriptor WHERE rangeURI = ?", PropertyType.FILE_LINK.getTypeUri()).getObject(Long.class)); + results.put("sampleTypeWithFileColumnCount", new SqlSelector(schema, """ + SELECT COUNT(DISTINCT DD.DomainURI) FROM + exp.PropertyDescriptor D\s + JOIN exp.PropertyDomain PD ON D.propertyId = PD.propertyid + JOIN exp.DomainDescriptor DD on PD.domainID = DD.domainId + WHERE DD.storageSchemaName = ? AND D.rangeURI = ?""", SampleTypeDomainKind.PROVISIONED_SCHEMA_NAME, PropertyType.FILE_LINK.getTypeUri()).getObject(Long.class)); + results.put("sampleTypeWithBooleanColumnCount", new SqlSelector(schema, """ + SELECT COUNT(DISTINCT DD.DomainURI) FROM + exp.PropertyDescriptor D\s + JOIN exp.PropertyDomain PD ON D.propertyId = PD.propertyid + JOIN exp.DomainDescriptor DD on PD.domainID = DD.domainId + WHERE DD.storageSchemaName = ? AND D.rangeURI = ?""", SampleTypeDomainKind.PROVISIONED_SCHEMA_NAME, PropertyType.BOOLEAN.getTypeUri()).getObject(Long.class)); + + results.put("sampleTypeAliquotSpecificField", new SqlSelector(schema, """ + SELECT COUNT(DISTINCT D.PropertyURI) FROM + exp.PropertyDescriptor D\s + JOIN exp.PropertyDomain PD ON D.propertyId = PD.propertyid + JOIN exp.DomainDescriptor DD on PD.domainID = DD.domainId + WHERE DD.storageSchemaName = ? AND D.derivationDataScope = ?""", SampleTypeDomainKind.PROVISIONED_SCHEMA_NAME, ExpSchema.DerivationDataScopeType.ChildOnly.name()).getObject(Long.class)); + results.put("sampleTypeParentOnlyField", new SqlSelector(schema, """ + SELECT COUNT(DISTINCT D.PropertyURI) FROM + exp.PropertyDescriptor D\s + JOIN exp.PropertyDomain PD ON D.propertyId = PD.propertyid + JOIN exp.DomainDescriptor DD on PD.domainID = DD.domainId + WHERE DD.storageSchemaName = ? AND (D.derivationDataScope = ? OR D.derivationDataScope IS NULL)""", SampleTypeDomainKind.PROVISIONED_SCHEMA_NAME, ExpSchema.DerivationDataScopeType.ParentOnly.name()).getObject(Long.class)); + results.put("sampleTypeParentAndAliquotField", new SqlSelector(schema, """ + SELECT COUNT(DISTINCT D.PropertyURI) FROM + exp.PropertyDescriptor D\s + JOIN exp.PropertyDomain PD ON D.propertyId = PD.propertyid + JOIN exp.DomainDescriptor DD on PD.domainID = DD.domainId + WHERE DD.storageSchemaName = ? AND D.derivationDataScope = ?""", SampleTypeDomainKind.PROVISIONED_SCHEMA_NAME, ExpSchema.DerivationDataScopeType.All.name()).getObject(Long.class)); + + results.put("attachmentColumnCount", new SqlSelector(schema, "SELECT COUNT(*) FROM exp.propertydescriptor WHERE rangeURI = ?", PropertyType.ATTACHMENT.getTypeUri()).getObject(Long.class)); + results.put("dataClassWithAttachmentColumnCount", new SqlSelector(schema, """ + SELECT COUNT(DISTINCT DD.DomainURI) FROM + exp.PropertyDescriptor D\s + JOIN exp.PropertyDomain PD ON D.propertyId = PD.propertyid + JOIN exp.DomainDescriptor DD on PD.domainID = DD.domainId + WHERE DD.storageSchemaName = ? AND D.rangeURI = ?""", DataClassDomainKind.PROVISIONED_SCHEMA_NAME, PropertyType.ATTACHMENT.getTypeUri()).getObject(Long.class)); + results.put("dataClassWithBooleanColumnCount", new SqlSelector(schema, """ + SELECT COUNT(DISTINCT DD.DomainURI) FROM + exp.PropertyDescriptor D\s + JOIN exp.PropertyDomain PD ON D.propertyId = PD.propertyid + JOIN exp.DomainDescriptor DD on PD.domainID = DD.domainId + WHERE DD.storageSchemaName = ? AND D.rangeURI = ?""", DataClassDomainKind.PROVISIONED_SCHEMA_NAME, PropertyType.BOOLEAN.getTypeUri()).getObject(Long.class)); + + results.put("textChoiceColumnCount", new SqlSelector(schema, "SELECT COUNT(*) FROM exp.propertydescriptor WHERE concepturi = ?", TEXT_CHOICE_CONCEPT_URI).getObject(Long.class)); + results.put("multiValueTextChoiceColumnCount", new SqlSelector(schema, "SELECT COUNT(*) FROM exp.propertydescriptor WHERE rangeuri = ?", PropertyType.MULTI_CHOICE.getTypeUri()).getObject(Long.class)); + + results.put("domainsWithDateTimeColumnCount", new SqlSelector(schema, """ + SELECT COUNT(DISTINCT DD.DomainURI) FROM + exp.PropertyDescriptor D\s + JOIN exp.PropertyDomain PD ON D.propertyId = PD.propertyid + JOIN exp.DomainDescriptor DD on PD.domainID = DD.domainId + WHERE D.rangeURI = ?""", PropertyType.DATE_TIME.getTypeUri()).getObject(Long.class)); + + results.put("domainsWithDateColumnCount", new SqlSelector(schema, """ + SELECT COUNT(DISTINCT DD.DomainURI) FROM + exp.PropertyDescriptor D\s + JOIN exp.PropertyDomain PD ON D.propertyId = PD.propertyid + JOIN exp.DomainDescriptor DD on PD.domainID = DD.domainId + WHERE D.rangeURI = ?""", PropertyType.DATE.getTypeUri()).getObject(Long.class)); + + results.put("domainsWithTimeColumnCount", new SqlSelector(schema, """ + SELECT COUNT(DISTINCT DD.DomainURI) FROM + exp.PropertyDescriptor D\s + JOIN exp.PropertyDomain PD ON D.propertyId = PD.propertyid + JOIN exp.DomainDescriptor DD on PD.domainID = DD.domainId + WHERE D.rangeURI = ?""", PropertyType.TIME.getTypeUri()).getObject(Long.class)); + + results.put("maxObjectObjectId", new SqlSelector(schema, "SELECT MAX(ObjectId) FROM exp.Object").getObject(Long.class)); + results.put("maxMaterialRowId", new SqlSelector(schema, "SELECT MAX(RowId) FROM exp.Material").getObject(Long.class)); + + results.putAll(ExperimentService.get().getDomainMetrics()); + + return results; + }); + } + + ExperimentMigrationSchemaHandler handler = new ExperimentMigrationSchemaHandler(); + DatabaseMigrationService.get().registerSchemaHandler(handler); + DatabaseMigrationService.get().registerTableHandler(new MigrationTableHandler() + { + @Override + public TableInfo getTableInfo() + { + return DbSchema.get("premium", DbSchemaType.Bare).getTable("Exclusions"); + } + + @Override + public void adjustFilter(TableInfo sourceTable, SimpleFilter filter, Set containers) + { + // Include experiment runs that were copied + FilterClause includedClause = handler.getIncludedRowIdClause(sourceTable, FieldKey.fromParts("RunId")); + if (includedClause != null) + filter.addClause(includedClause); + } + }); + DatabaseMigrationService.get().registerTableHandler(new MigrationTableHandler() + { + @Override + public TableInfo getTableInfo() + { + return DbSchema.get("premium", DbSchemaType.Bare).getTable("ExclusionMaps"); + } + + @Override + public void adjustFilter(TableInfo sourceTable, SimpleFilter filter, Set containers) + { + // Include experiment runs that were copied + FilterClause includedClause = handler.getIncludedRowIdClause(sourceTable, FieldKey.fromParts("ExclusionId", "RunId")); + if (includedClause != null) + filter.addClause(includedClause); + } + }); + DatabaseMigrationService.get().registerTableHandler(new MigrationTableHandler() + { + @Override + public TableInfo getTableInfo() + { + return DbSchema.get("assayrequest", DbSchemaType.Bare).getTable("RequestRunsJunction"); + } + + @Override + public void adjustFilter(TableInfo sourceTable, SimpleFilter filter, Set containers) + { + // Include experiment runs that were copied + FilterClause includedClause = handler.getIncludedRowIdClause(sourceTable, FieldKey.fromParts("RunId")); + if (includedClause != null) + filter.addClause(includedClause); + } + }); + DatabaseMigrationService.get().registerSchemaHandler(new SampleTypeMigrationSchemaHandler()); + DataClassMigrationSchemaHandler dcHandler = new DataClassMigrationSchemaHandler(); + DatabaseMigrationService.get().registerSchemaHandler(dcHandler); + ExperimentDeleteService.setInstance(dcHandler); + } + + @Override + @NotNull + public Collection getSummary(Container c) + { + Collection list = new LinkedList<>(); + int runGroupCount = ExperimentService.get().getExperiments(c, null, false, true).size(); + if (runGroupCount > 0) + list.add(StringUtilsLabKey.pluralize(runGroupCount, "Run Group")); + + User user = HttpView.currentContext().getUser(); + + Set runTypes = ExperimentService.get().getExperimentRunTypes(c); + for (ExperimentRunType runType : runTypes) + { + if (runType == ExperimentRunType.ALL_RUNS_TYPE) + continue; + + long runCount = runType.getRunCount(user, c); + if (runCount > 0) + list.add(runCount + " runs of type " + runType.getDescription()); + } + + int dataClassCount = ExperimentService.get().getDataClasses(c, false).size(); + if (dataClassCount > 0) + list.add(dataClassCount + " Data Class" + (dataClassCount > 1 ? "es" : "")); + + int sampleTypeCount = SampleTypeService.get().getSampleTypes(c, false).size(); + if (sampleTypeCount > 0) + list.add(sampleTypeCount + " Sample Type" + (sampleTypeCount > 1 ? "s" : "")); + + return list; + } + + @Override + public @NotNull ArrayList getDetailedSummary(Container c, User user) + { + ArrayList summaries = new ArrayList<>(); + + // Assay types + long assayTypeCount = AssayService.get().getAssayProtocols(c).stream().filter(p -> p.getContainer().equals(c)).count(); + if (assayTypeCount > 0) + summaries.add(new Summary(assayTypeCount, "Assay Type")); + + // Run count + int runGroupCount = ExperimentService.get().getExperiments(c, user, false, true).size(); + if (runGroupCount > 0) + summaries.add(new Summary(runGroupCount, "Assay run")); + + // Number of Data Classes + List dataClasses = ExperimentService.get().getDataClasses(c, false); + int dataClassCount = dataClasses.size(); + if (dataClassCount > 0) + summaries.add(new Summary(dataClassCount, "Data Class")); + + ExpSchema expSchema = new ExpSchema(user, c); + + // Individual Data Class row counts + { + // The table-level container filter is set to ensure data class types are included + // that may not be defined in the target container but may have rows of data in the target container + TableInfo table = ExpSchema.TableType.DataClasses.createTable(expSchema, null, ContainerFilter.Type.CurrentPlusProjectAndShared.create(c, user)); + + // Issue 47919: The "DataCount" column is filtered to only count data in the target container + if (table instanceof ExpDataClassTableImpl tableImpl) + tableImpl.setDataCountContainerFilter(ContainerFilter.Type.Current.create(c, user)); + + Set columns = new LinkedHashSet<>(); + columns.add(ExpDataClassTable.Column.Name.name()); + columns.add(ExpDataClassTable.Column.DataCount.name()); + + Map results = new TableSelector(table, columns).getValueMap(String.class); + for (var entry : results.entrySet()) + { + long count = entry.getValue().longValue(); + if (count > 0) + summaries.add(new Summary(count, entry.getKey())); + } + } + + // Sample Types + int sampleTypeCount = SampleTypeService.get().getSampleTypes(c, false).size(); + if (sampleTypeCount > 0) + summaries.add(new Summary(sampleTypeCount, "Sample Type")); + + // Individual Sample Type row counts + { + // The table-level container filter is set to ensure data class types are included + // that may not be defined in the target container but may have rows of data in the target container + TableInfo table = ExpSchema.TableType.SampleSets.createTable(expSchema, null, ContainerFilter.Type.CurrentPlusProjectAndShared.create(c, user)); + + // Issue 51557: The "SampleCount" column is filtered to only count data in the target container + if (table instanceof ExpSampleTypeTableImpl tableImpl) + tableImpl.setSampleCountContainerFilter(ContainerFilter.Type.Current.create(c, user)); + + Set columns = new LinkedHashSet<>(); + columns.add(ExpSampleTypeTable.Column.Name.name()); + columns.add(ExpSampleTypeTable.Column.SampleCount.name()); + + Map results = new TableSelector(table, columns).getValueMap(String.class); + for (var entry : results.entrySet()) + { + long count = entry.getValue().longValue(); + if (count > 0) + { + String name = entry.getKey(); + Summary s = name.equals("MixtureBatches") + ? new Summary(count, "Batch") + : new Summary(count, name); + summaries.add(s); + } + } + } + + return summaries; + } + + @Override + public @NotNull Set> getIntegrationTests() + { + return Set.of( + DomainImpl.TestCase.class, + DomainPropertyImpl.TestCase.class, + ExpDataTableImpl.TestCase.class, + ExperimentServiceImpl.AuditDomainUriTest.class, + ExperimentServiceImpl.LineageQueryTestCase.class, + ExperimentServiceImpl.ParseInputOutputAliasTestCase.class, + ExperimentServiceImpl.TestCase.class, + ExperimentStressTest.class, + LineagePerfTest.class, + LineageTest.class, + OntologyManager.TestCase.class, + PropertyServiceImpl.TestCase.class, + SampleTypeServiceImpl.TestCase.class, + StorageNameGenerator.TestCase.class, + StorageProvisionerImpl.TestCase.class, + UniqueValueCounterTestCase.class, + XarTestPipelineJob.TestCase.class + ); + } + + @Override + public @NotNull Collection>> getIntegrationTestFactories() + { + List>> list = new ArrayList<>(super.getIntegrationTestFactories()); + list.add(new JspTestCase("/org/labkey/experiment/api/ExpDataClassDataTestCase.jsp")); + list.add(new JspTestCase("/org/labkey/experiment/api/ExpSampleTypeTestCase.jsp")); + return list; + } + + @Override + public @NotNull Set> getUnitTests() + { + return Set.of( + GraphAlgorithms.TestCase.class, + LSIDRelativizer.TestCase.class, + Lsid.TestCase.class, + LsidUtils.TestCase.class, + PropertyController.TestCase.class, + Quantity.TestCase.class, + Unit.TestCase.class + ); + } + + @Override + @NotNull + public Collection getSchemaNames() + { + return List.of( + ExpSchema.SCHEMA_NAME, + DataClassDomainKind.PROVISIONED_SCHEMA_NAME, + SampleTypeDomainKind.PROVISIONED_SCHEMA_NAME + ); + } + + @NotNull + @Override + public Collection getProvisionedSchemaNames() + { + return PageFlowUtil.set(DataClassDomainKind.PROVISIONED_SCHEMA_NAME, SampleTypeDomainKind.PROVISIONED_SCHEMA_NAME); + } +} diff --git a/experiment/src/org/labkey/experiment/api/DataClassDomainKind.java b/experiment/src/org/labkey/experiment/api/DataClassDomainKind.java index e440a6fb275..af1d3f26651 100644 --- a/experiment/src/org/labkey/experiment/api/DataClassDomainKind.java +++ b/experiment/src/org/labkey/experiment/api/DataClassDomainKind.java @@ -204,6 +204,12 @@ public boolean allowAttachmentProperties() return true; } + @Override + public boolean allowMultiChoiceProperties() + { + return true; + } + @Override public ActionURL urlCreateDefinition(String schemaName, String queryName, Container container, User user) { diff --git a/list/src/org/labkey/list/model/ListDomainKind.java b/list/src/org/labkey/list/model/ListDomainKind.java index acf70af615a..f29b9169164 100644 --- a/list/src/org/labkey/list/model/ListDomainKind.java +++ b/list/src/org/labkey/list/model/ListDomainKind.java @@ -1,744 +1,750 @@ -/* - * Copyright (c) 2013-2018 LabKey Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.labkey.list.model; - -import org.apache.commons.lang3.EnumUtils; -import org.apache.commons.lang3.StringUtils; -import org.jetbrains.annotations.NotNull; -import org.jetbrains.annotations.Nullable; -import org.labkey.api.action.ApiUsageException; -import org.labkey.api.attachments.AttachmentService; -import org.labkey.api.collections.CaseInsensitiveHashMap; -import org.labkey.api.collections.CaseInsensitiveHashSet; -import org.labkey.api.compliance.ComplianceService; -import org.labkey.api.data.ColumnInfo; -import org.labkey.api.data.Container; -import org.labkey.api.data.ContainerFilter; -import org.labkey.api.data.DbScope; -import org.labkey.api.data.JdbcType; -import org.labkey.api.data.PropertyStorageSpec; -import org.labkey.api.data.RuntimeSQLException; -import org.labkey.api.data.SQLFragment; -import org.labkey.api.data.SimpleFilter; -import org.labkey.api.data.TableInfo; -import org.labkey.api.data.TableSelector; -import org.labkey.api.defaults.DefaultValueService; -import org.labkey.api.di.DataIntegrationService; -import org.labkey.api.exp.DomainNotFoundException; -import org.labkey.api.exp.Lsid; -import org.labkey.api.exp.LsidManager; -import org.labkey.api.exp.OntologyManager; -import org.labkey.api.exp.PropertyDescriptor; -import org.labkey.api.exp.PropertyType; -import org.labkey.api.exp.TemplateInfo; -import org.labkey.api.exp.api.ExperimentService; -import org.labkey.api.exp.list.ListDefinition; -import org.labkey.api.exp.list.ListDefinition.KeyType; -import org.labkey.api.exp.list.ListService; -import org.labkey.api.exp.list.ListUrls; -import org.labkey.api.exp.property.AbstractDomainKind; -import org.labkey.api.exp.property.Domain; -import org.labkey.api.exp.property.DomainProperty; -import org.labkey.api.exp.property.DomainUtil; -import org.labkey.api.exp.property.PropertyService; -import org.labkey.api.gwt.client.model.GWTDomain; -import org.labkey.api.gwt.client.model.GWTIndex; -import org.labkey.api.gwt.client.model.GWTPropertyDescriptor; -import org.labkey.api.lists.permissions.DesignListPermission; -import org.labkey.api.query.FieldKey; -import org.labkey.api.query.MetadataUnavailableException; -import org.labkey.api.query.QueryService; -import org.labkey.api.query.ValidationException; -import org.labkey.api.security.User; -import org.labkey.api.settings.AppProps; -import org.labkey.api.util.PageFlowUtil; -import org.labkey.api.util.Pair; -import org.labkey.api.view.ActionURL; -import org.labkey.api.view.NotFoundException; -import org.labkey.api.writer.ContainerUser; -import org.labkey.data.xml.domainTemplate.DomainTemplateType; -import org.labkey.data.xml.domainTemplate.ListOptionsType; -import org.labkey.data.xml.domainTemplate.ListTemplateType; -import org.labkey.list.view.ListItemAttachmentParent; -import org.springframework.dao.DataIntegrityViolationException; - -import java.util.ArrayList; -import java.util.Collection; -import java.util.Collections; -import java.util.HashMap; -import java.util.HashSet; -import java.util.List; -import java.util.Map; -import java.util.Set; -import java.util.stream.Collectors; - -import static org.labkey.api.exp.property.DomainTemplate.findProperty; - -public abstract class ListDomainKind extends AbstractDomainKind -{ - /* - * the columns common to all lists - */ - private final static Set BASE_PROPERTIES; - private ListDefinitionImpl _list; - private final static int MAX_NAME_LENGTH = 200; - private static final Set RESERVED_NAMES; - - static - { - BASE_PROPERTIES = PageFlowUtil.set(new PropertyStorageSpec("entityId", JdbcType.GUID).setNullable(false), - new PropertyStorageSpec("created", JdbcType.TIMESTAMP), - new PropertyStorageSpec("createdBy", JdbcType.INTEGER), - new PropertyStorageSpec("modified", JdbcType.TIMESTAMP), - new PropertyStorageSpec("modifiedBy", JdbcType.INTEGER), - new PropertyStorageSpec("lastIndexed", JdbcType.TIMESTAMP), - new PropertyStorageSpec("container", JdbcType.GUID).setNullable(false), - new PropertyStorageSpec(DataIntegrationService.Columns.TransformImportHash.getColumnName(), JdbcType.VARCHAR, 256)); - RESERVED_NAMES = DomainUtil.getNamesAndLabels( - BASE_PROPERTIES.stream().map(PropertyStorageSpec::getName).collect(Collectors.toSet())); - } - - public void setListDefinition(ListDefinitionImpl list) - { - _list = list; - } - - @Override - public String getTypeLabel(Domain domain) - { - return "List '" + domain.getName() + "'"; - } - - @Override - public SQLFragment sqlObjectIdsInDomain(Domain domain) - { - throw new UnsupportedOperationException("sqlObjectIdsInDomain NYI for ListDomainKind"); - } - - @Override - public ActionURL urlShowData(Domain domain, ContainerUser containerUser) - { - ListDefinition listDef = ListService.get().getList(domain); - if (null == listDef) - return null; - return listDef.urlShowData(); - } - - @Nullable - @Override - public ActionURL urlEditDefinition(Domain domain, ContainerUser containerUser) - { - ListDefinition listDef = ListService.get().getList(domain); - if (null == listDef) - return null; - return listDef.urlShowDefinition(); - } - - @Override - public boolean allowAttachmentProperties() - { - return true; - } - - @Override - public boolean showDefaultValueSettings() - { - return true; - } - - @Override - public boolean allowUniqueConstraintProperties() - { - return true; - } - - @Override - public boolean allowCalculatedFields() - { - return true; - } - - @Override - public Priority getPriority(String domainURI) - { - Lsid lsid = new Lsid(domainURI); - return getKindName().equals(lsid.getNamespacePrefix()) ? Priority.MEDIUM : null; - } - - - @Override - public Set getBaseProperties(Domain domain) - { - Set specs = new HashSet<>(BASE_PROPERTIES); - specs.addAll(super.getBaseProperties(domain)); - return specs; - } - - @Override - public Set getMandatoryPropertyNames(Domain domain) - { - Set properties = super.getMandatoryPropertyNames(domain); - properties.addAll(getAdditionalProtectedPropertyNames(domain)); - return properties; - } - - /** - * Returns the List's primary key as a field to get special treatment elsewhere, despite being property driven. - */ - @Override - public Set getAdditionalProtectedProperties(Domain domain) - { - Set specs = new HashSet<>(super.getAdditionalProtectedProperties(domain)); - - ListDefinition listDef = ListService.get().getList(domain); - if (null != listDef) - { - String keyName = listDef.getKeyName(); - JdbcType keyType = listDef.getKeyType() == KeyType.Varchar ? JdbcType.VARCHAR : JdbcType.INTEGER; - int keySize = keyType == JdbcType.VARCHAR ? 4000 : 0; - specs.add(new PropertyStorageSpec(keyName, keyType, keySize, PropertyStorageSpec.Special.PrimaryKey)); - } - - return specs; - } - - @Override - public PropertyStorageSpec getPropertySpec(PropertyDescriptor pd, Domain domain) - { - ListDefinition list = ListService.get().getList(domain); - if (null == list) - list = _list; - - if (null != list) - { - if (pd.getName().equalsIgnoreCase(list.getKeyName())) - { - PropertyStorageSpec key = getKeyProperty(list, pd.getStorageColumnName()); - assert key.isPrimaryKey(); - _list = null; - return key; - } - } - return super.getPropertySpec(pd, domain); - } - - abstract PropertyStorageSpec getKeyProperty(ListDefinition list, String storageColumnName); - - abstract KeyType getDefaultKeyType(); - - abstract Collection getSupportedKeyTypes(); - - @Override - public @NotNull Set getReservedPropertyNames(Domain domain, User user) - { - return RESERVED_NAMES; - } - - @Override - public String getStorageSchemaName() - { - return ListSchema.getInstance().getSchemaName(); - } - - @Override - public DbScope getScope() - { - return ListSchema.getInstance().getSchema().getScope(); - } - - @Override - public Set getPropertyIndices(Domain domain) - { - // return PageFlowUtil.set(new PropertyStorageSpec.Index(false, ListDomainKind.KEY_FIELD)); - return Collections.emptySet(); // TODO: Allow this to return the Key Column - } - - public static Lsid generateDomainURI(Container c, KeyType keyType, @Nullable ListDefinition.Category category) - { - String type = getType(keyType, category); - Lsid lsid; - // assure LSID does not collide with previous lsids that may have had number names - do - { - String dbSeqStr = String.valueOf(LsidManager.getLsidPrefixDbSeq(type, 1).next()); - lsid = new Lsid(type, "Folder-" + c.getRowId(), dbSeqStr); - } while (OntologyManager.getDomainDescriptor(lsid.toString(), c) != null); - - return lsid; - } - - public static Lsid createPropertyURI(String listName, String columnName, Container c, ListDefinition.KeyType keyType) - { - StringBuilder typeURI = getBaseURI(listName, getType(keyType, null), c); - typeURI.append(".").append(PageFlowUtil.encode(columnName)); - return new Lsid(typeURI.toString()); - } - - private static String getType(KeyType keyType, @Nullable ListDefinition.Category category) - { - String type; - switch (keyType) - { - case Integer: - case AutoIncrementInteger: - if (category != null) - type = PicklistDomainKind.NAMESPACE_PREFIX; - else - type = IntegerListDomainKind.NAMESPACE_PREFIX; - break; - case Varchar: - type = VarcharListDomainKind.NAMESPACE_PREFIX; - break; - default: - throw new IllegalStateException(); - } - return type; - } - - private static StringBuilder getBaseURI(String listName, String type, Container c) - { - return new StringBuilder("urn:lsid:") - .append(PageFlowUtil.encode(AppProps.getInstance().getDefaultLsidAuthority())) - .append(":").append(type).append(".Folder-").append(c.getRowId()).append(":") - .append(PageFlowUtil.encode(listName)); - } - - @Override - public TableInfo getTableInfo(User user, Container container, String name, @Nullable ContainerFilter cf) - { - return new ListQuerySchema(user, container).createTable(name, cf); - } - - @Override - public boolean isDeleteAllDataOnFieldImport() - { - return true; - } - - @Override - public boolean canCreateDefinition(User user, Container container) - { - return container.hasPermission("ListDomainKind.canCreateDefinition", user, DesignListPermission.class); - } - - @Override - public boolean canEditDefinition(User user, Domain domain) - { - Container c = domain.getContainer(); - return c.hasPermission("ListDomainKind.canEditDefinition", user, DesignListPermission.class); - } - - @Override - public ActionURL urlCreateDefinition(String schemaName, String queryName, Container container, User user) - { - return PageFlowUtil.urlProvider(ListUrls.class).getCreateListURL(container); - } - - @Override - public Class getTypeClass() - { - return ListDomainKindProperties.class; - } - - @Override - public void validateDomainName(Container container, User user, @Nullable Domain domain, String name) - { - if (StringUtils.isEmpty(name)) - throw new ApiUsageException("List name is required."); - if (name.length() > MAX_NAME_LENGTH) - throw new ApiUsageException("List name cannot be longer than " + MAX_NAME_LENGTH + " characters."); - if (ListService.get().getList(container, name, domain == null) != null) - throw new ApiUsageException("The name '" + name + "' is already in use."); - } - - @Override - public Domain createDomain(GWTDomain domain, ListDomainKindProperties listProperties, Container container, User user, @Nullable TemplateInfo templateInfo, boolean forUpdate) - { - String name = StringUtils.trimToEmpty(domain.getName()); - validateDomainName(container, user, null, name); - - String keyName = listProperties.getKeyName(); - if (StringUtils.isEmpty(keyName)) - throw new ApiUsageException("List keyName is required"); - - KeyType keyType = getDefaultKeyType(); - - if (null != listProperties.getKeyType()) - { - String rawKeyType = listProperties.getKeyType(); - if (EnumUtils.isValidEnum(KeyType.class, rawKeyType)) - keyType = KeyType.valueOf(rawKeyType); - else - throw new ApiUsageException("List keyType provided does not exist."); - } - - ListDefinition.Category category; - try - { - category = listProperties.getCategory() != null ? ListDefinition.Category.valueOf(listProperties.getCategory()) : null; - } - catch (IllegalArgumentException e) - { - throw new ApiUsageException(String.format("List category type provided (%s) does not exist.", listProperties.getCategory())); - } - - if (!getSupportedKeyTypes().contains(keyType)) - throw new ApiUsageException("List keyType provided is not supported for list domain kind (" + getKindName() + ")."); - - ListDefinition list = ListService.get().createList(container, name, keyType, templateInfo, category); - list.setKeyName(keyName); - - // Issue 45042: Allow for the list description to be set via the create domain API calls - if (listProperties.getDescription() == null && domain.getDescription() != null) - listProperties.setDescription(domain.getDescription()); - list.setDescription(listProperties.getDescription()); - - List properties = (List)domain.getFields(); - List indices = (List)domain.getIndices(); - - try (DbScope.Transaction tx = ExperimentService.get().ensureTransaction()) - { - Domain d = list.getDomain(forUpdate); - - Set reservedNames = getReservedPropertyNames(d, user); - Set lowerReservedNames = reservedNames.stream().map(String::toLowerCase).collect(Collectors.toSet()); - - Map defaultValues = new HashMap<>(); - Set propertyUris = new CaseInsensitiveHashSet(); - for (GWTPropertyDescriptor pd : properties) - { - String propertyName = pd.getName().toLowerCase(); - if (lowerReservedNames.contains(propertyName)) - { - if (pd.getLabel() == null) - pd.setLabel(pd.getName()); - pd.setName("_" + pd.getName()); - } - - DomainUtil.addProperty(d, pd, defaultValues, propertyUris, null); - } - - d.setPropertyIndices(indices, lowerReservedNames); - - // TODO: This looks like the wrong order to me -- we should updateListProperties() (persist the indexing - // settings and handle potential transitions) before calling save() (which indexes the list). Since this is - // the create case there's no data to index, but there is meta data... - list.save(user, true, listProperties.getAuditRecordMap(), domain.getCalculatedFields()); - updateListProperties(container, user, list.getListId(), listProperties); - - QueryService.get().saveCalculatedFieldsMetadata(ListQuerySchema.NAME, name, null, domain.getCalculatedFields(), false, user, container); - - DefaultValueService.get().setDefaultValues(container, defaultValues); - - tx.commit(); - } - catch (Exception e) - { - throw new RuntimeException(e); - } - - return list.getDomain(forUpdate); - } - - @Override - public @NotNull ValidationException updateDomain(GWTDomain original, GWTDomain update, - ListDomainKindProperties listProperties, Container container, User user, boolean includeWarnings, @Nullable String auditUserComment) - { - ValidationException exception; - - try (DbScope.Transaction transaction = ListManager.get().getListMetadataSchema().getScope().ensureTransaction()) - { - exception = new ValidationException(); - StringBuilder changeDetails = new StringBuilder(); - - Domain domain = PropertyService.get().getDomain(container, original.getDomainURI()); - if (null == domain) - return exception.addGlobalError("Expected domain for list: " + original.getName()); - - ListDefinition listDefinition = ListService.get().getList(domain); - TableInfo table = listDefinition.getTable(user); - - if (null == table) - return exception.addGlobalError("Expected table for list: " + listDefinition.getName()); - - // Handle cases when existing key field is null or is not provided in the updated domainDesign - GWTPropertyDescriptor key = findField(listDefinition.getKeyName(), original.getFields()); - if (null != key) - { - int id = key.getPropertyId(); - GWTPropertyDescriptor newKey = findField(id, update.getFields()); - if (null == newKey) - { - return exception.addGlobalError("Key field not provided, expecting key field '" + key.getName() + "'"); - } - else if (!key.getName().equalsIgnoreCase(newKey.getName())) - { - return exception.addGlobalError("Cannot change key field name"); - } - } - else - { - return exception.addGlobalError("Key field not found for list '" + listDefinition.getName() + "'"); - } - - //handle name change - String updatedName = StringUtils.trim(update.getName()); - boolean hasNameChange = !original.getName().equals(updatedName); - if (hasNameChange) - { - validateDomainName(container, user, domain, updatedName); - changeDetails.append("The name of the list domain '" + original.getName() + "' was changed to '" + updatedName + "'."); - } - - //return if there are errors before moving forward with the save - if (exception.hasErrors()) - { - return exception; - } - - //update list properties - Map oldProps = null; - Map newProps = null; - if (null != listProperties) - { - if (listProperties.getDomainId() != original.getDomainId() || listProperties.getDomainId() != update.getDomainId()) - return exception.addGlobalError("domainId for the list does not match old or the new domain"); - if (!original.getDomainURI().equals(update.getDomainURI())) - return exception.addGlobalError("domainURI mismatch between old and new domain"); - - Pair, Map> updatedProps = updateListProperties(container, user, listDefinition.getListId(), listProperties); - oldProps = updatedProps.first; - newProps = updatedProps.second; - } - // Issue 45042: Allow for the list description to be set via the save domain API calls - else if (update.getDescription() != null) - { - listProperties = getListProperties(container, user, listDefinition.getListId()); - listProperties.setDescription(update.getDescription()); - Pair, Map> updatedProps = updateListProperties(container, user, listDefinition.getListId(), listProperties); - oldProps = updatedProps.first; - newProps = updatedProps.second; - } - - //update domain design properties - try - { - //handle attachment cols - Map modifiedAttachmentColumns = new CaseInsensitiveHashMap<>(); - - for (DomainProperty oldProp : domain.getProperties()) - { - if (PropertyType.ATTACHMENT.equals(oldProp.getPropertyDescriptor().getPropertyType())) - { - GWTPropertyDescriptor newGWTProp = findField(oldProp.getPropertyId(), update.getFields()); - if (null == newGWTProp || !PropertyType.ATTACHMENT.equals(PropertyType.getFromURI(newGWTProp.getConceptURI(), newGWTProp.getRangeURI(), null))) - { - ColumnInfo column = table.getColumn(oldProp.getPropertyDescriptor().getName()); - if (null != column) - modifiedAttachmentColumns.put(oldProp.getPropertyDescriptor().getName(), column); - } - } - } - - Collection> attachmentMapCollection = null; - if (!modifiedAttachmentColumns.isEmpty()) - { - List columns = new ArrayList<>(modifiedAttachmentColumns.values()); - columns.add(table.getColumn("entityId")); - attachmentMapCollection = new TableSelector(table, columns, null, null).getMapCollection(); - } - - // Remove attachments from any attachment columns that are removed or no longer attachment columns - if (null != attachmentMapCollection) - { - for (Map map : attachmentMapCollection) - { - String entityId = (String) map.get("entityId"); - ListItemAttachmentParent parent = new ListItemAttachmentParent(entityId, container); - for (Map.Entry entry : map.entrySet()) - if (null != entry.getValue() && modifiedAttachmentColumns.containsKey(entry.getKey())) - AttachmentService.get().deleteAttachment(parent, entry.getValue().toString(), user); - } - } - - //update domain properties - exception.addErrors(DomainUtil.updateDomainDescriptor(original, update, container, user, hasNameChange, changeDetails.toString(), auditUserComment, oldProps, newProps)); - - QueryService.get().saveCalculatedFieldsMetadata(ListQuerySchema.NAME, update.getQueryName(), hasNameChange ? update.getName() : null, update.getCalculatedFields(), !original.getCalculatedFields().isEmpty(), user, container); - } - catch (RuntimeSQLException x) - { - // issue 19202 - check for null value exceptions in case provided file data not contain the column - // and return a better error message - String message = x.getMessage(); - if (x.isNullValueException()) - { - message = "The provided data does not contain the specified '" + listDefinition.getKeyName() + "' field or contains null key values."; - } - return exception.addGlobalError(message); - } - catch (DataIntegrityViolationException | MetadataUnavailableException x) - { - return exception.addGlobalError("A data error occurred: " + x.getMessage()); - } - - if (!exception.hasErrors()) - { - transaction.commit(); - } - return exception; - } - } - - private ListDomainKindProperties getListProperties(Container container, User user, int listId) - { - SimpleFilter filter = SimpleFilter.createContainerFilter(container); - filter.addCondition(FieldKey.fromParts("ListId"), listId); - return new TableSelector(ListManager.get().getListMetadataTable(), filter, null).getObject(ListDomainKindProperties.class); - } - - private Pair, Map> updateListProperties(Container container, User user, int listId, ListDomainKindProperties listProperties) - { - ListDomainKindProperties existingListProps = getListProperties(container, user, listId); - - Map oldProps = existingListProps == null ? null : existingListProps.getAuditRecordMap(); - Map newProps = listProperties == null ? oldProps : listProperties.getAuditRecordMap(); - //merge existing and new properties - ListDomainKindProperties updatedListProps = updateListProperties(existingListProps, listProperties); - - ListManager.get().update(user, container, updatedListProps); - - return new Pair<>(oldProps, newProps); - } - - //updates list properties except listId, domainId, keyName, keyType, and lastIndexed - private ListDomainKindProperties updateListProperties(ListDomainKindProperties existingListProps, ListDomainKindProperties newListProps) - { - - ListDomainKindProperties updatedListProps = new ListDomainKindProperties(existingListProps); - if (null != newListProps.getName()) - updatedListProps.setName(newListProps.getName().trim()); - - updatedListProps.setTitleColumn(newListProps.getTitleColumn()); - updatedListProps.setDescription(newListProps.getDescription()); - updatedListProps.setAllowDelete(newListProps.isAllowDelete()); - updatedListProps.setAllowUpload(newListProps.isAllowUpload()); - updatedListProps.setAllowExport(newListProps.isAllowExport()); - updatedListProps.setCategory(newListProps.getCategory()); - updatedListProps.setEntireListTitleTemplate(newListProps.getEntireListTitleTemplate()); - updatedListProps.setEntireListIndexSetting(newListProps.getEntireListIndexSetting()); - updatedListProps.setEntireListBodySetting(newListProps.getEntireListBodySetting()); - updatedListProps.setEachItemTitleTemplate(newListProps.getEachItemTitleTemplate()); - updatedListProps.setEachItemBodySetting(newListProps.getEachItemBodySetting()); - updatedListProps.setEntireListIndex(newListProps.isEntireListIndex()); - updatedListProps.setEntireListBodyTemplate(newListProps.getEntireListBodyTemplate()); - updatedListProps.setEachItemIndex(newListProps.isEachItemIndex()); - updatedListProps.setEachItemBodyTemplate(newListProps.getEachItemBodyTemplate()); - updatedListProps.setFileAttachmentIndex(newListProps.isFileAttachmentIndex()); - - return updatedListProps; - } - - private GWTPropertyDescriptor findField(String name, List fields) - { - for (GWTPropertyDescriptor f : fields) - { - if (name.equalsIgnoreCase(f.getName())) - return f; - } - return null; - } - - private GWTPropertyDescriptor findField(int id, List fields) - { - if (id > 0) - { - for (GWTPropertyDescriptor f : fields) - { - if (id == f.getPropertyId()) - return f; - } - } - return null; - } - - @Override - public @Nullable ListDomainKindProperties getDomainKindProperties(GWTDomain domain, Container container, User user) - { - ListDefinition list = domain != null ? ListService.get().getList(PropertyService.get().getDomain(domain.getDomainId())) : null; - return ListManager.get().getListDomainKindProperties(container, list != null ? list.getListId() : null); - } - - @Override - public void deleteDomain(User user, Domain domain, String userComment) - { - ListDefinition list = ListService.get().getList(domain); - if (list == null) - throw new NotFoundException("List not found: " + domain.getTypeURI()); - - try - { - list.delete(user, userComment); - } - catch (DomainNotFoundException e) - { - throw new NotFoundException(e.getMessage()); - } - } - - @Override - public void invalidate(Domain domain) - { - super.invalidate(domain); - - ListDefinition list = ListService.get().getList(domain); - if (list != null) - ListManager.get().indexList(list); - } - - @Override - public boolean matchesTemplateXML(String templateName, DomainTemplateType template, List properties) - { - if (!(template instanceof ListTemplateType)) - return false; - - ListOptionsType options = ((ListTemplateType) template).getOptions(); - if (options == null) - throw new IllegalArgumentException("List template requires specifying a keyCol"); - - String keyName = options.getKeyCol(); - if (keyName == null) - throw new IllegalArgumentException("List template requires specifying a keyCol"); - - Pair pair = findProperty(templateName, properties, keyName); - GWTPropertyDescriptor prop = pair.first; - - PropertyType type = PropertyType.getFromURI(prop.getConceptURI(), prop.getRangeURI()); - - return type.equals(getDefaultKeyType().getPropertyType()); - } - - public void ensureBaseProperties(Domain d) - { - var props = getBaseProperties(d); - } - - @Override - public boolean supportsPhiLevel() - { - return ComplianceService.get().isComplianceSupported(); - } -} +/* + * Copyright (c) 2013-2018 LabKey Corporation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.labkey.list.model; + +import org.apache.commons.lang3.EnumUtils; +import org.apache.commons.lang3.StringUtils; +import org.jetbrains.annotations.NotNull; +import org.jetbrains.annotations.Nullable; +import org.labkey.api.action.ApiUsageException; +import org.labkey.api.attachments.AttachmentService; +import org.labkey.api.collections.CaseInsensitiveHashMap; +import org.labkey.api.collections.CaseInsensitiveHashSet; +import org.labkey.api.compliance.ComplianceService; +import org.labkey.api.data.ColumnInfo; +import org.labkey.api.data.Container; +import org.labkey.api.data.ContainerFilter; +import org.labkey.api.data.DbScope; +import org.labkey.api.data.JdbcType; +import org.labkey.api.data.PropertyStorageSpec; +import org.labkey.api.data.RuntimeSQLException; +import org.labkey.api.data.SQLFragment; +import org.labkey.api.data.SimpleFilter; +import org.labkey.api.data.TableInfo; +import org.labkey.api.data.TableSelector; +import org.labkey.api.defaults.DefaultValueService; +import org.labkey.api.di.DataIntegrationService; +import org.labkey.api.exp.DomainNotFoundException; +import org.labkey.api.exp.Lsid; +import org.labkey.api.exp.LsidManager; +import org.labkey.api.exp.OntologyManager; +import org.labkey.api.exp.PropertyDescriptor; +import org.labkey.api.exp.PropertyType; +import org.labkey.api.exp.TemplateInfo; +import org.labkey.api.exp.api.ExperimentService; +import org.labkey.api.exp.list.ListDefinition; +import org.labkey.api.exp.list.ListDefinition.KeyType; +import org.labkey.api.exp.list.ListService; +import org.labkey.api.exp.list.ListUrls; +import org.labkey.api.exp.property.AbstractDomainKind; +import org.labkey.api.exp.property.Domain; +import org.labkey.api.exp.property.DomainProperty; +import org.labkey.api.exp.property.DomainUtil; +import org.labkey.api.exp.property.PropertyService; +import org.labkey.api.gwt.client.model.GWTDomain; +import org.labkey.api.gwt.client.model.GWTIndex; +import org.labkey.api.gwt.client.model.GWTPropertyDescriptor; +import org.labkey.api.lists.permissions.DesignListPermission; +import org.labkey.api.query.FieldKey; +import org.labkey.api.query.MetadataUnavailableException; +import org.labkey.api.query.QueryService; +import org.labkey.api.query.ValidationException; +import org.labkey.api.security.User; +import org.labkey.api.settings.AppProps; +import org.labkey.api.util.PageFlowUtil; +import org.labkey.api.util.Pair; +import org.labkey.api.view.ActionURL; +import org.labkey.api.view.NotFoundException; +import org.labkey.api.writer.ContainerUser; +import org.labkey.data.xml.domainTemplate.DomainTemplateType; +import org.labkey.data.xml.domainTemplate.ListOptionsType; +import org.labkey.data.xml.domainTemplate.ListTemplateType; +import org.labkey.list.view.ListItemAttachmentParent; +import org.springframework.dao.DataIntegrityViolationException; + +import java.util.ArrayList; +import java.util.Collection; +import java.util.Collections; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.stream.Collectors; + +import static org.labkey.api.exp.property.DomainTemplate.findProperty; + +public abstract class ListDomainKind extends AbstractDomainKind +{ + /* + * the columns common to all lists + */ + private final static Set BASE_PROPERTIES; + private ListDefinitionImpl _list; + private final static int MAX_NAME_LENGTH = 200; + private static final Set RESERVED_NAMES; + + static + { + BASE_PROPERTIES = PageFlowUtil.set(new PropertyStorageSpec("entityId", JdbcType.GUID).setNullable(false), + new PropertyStorageSpec("created", JdbcType.TIMESTAMP), + new PropertyStorageSpec("createdBy", JdbcType.INTEGER), + new PropertyStorageSpec("modified", JdbcType.TIMESTAMP), + new PropertyStorageSpec("modifiedBy", JdbcType.INTEGER), + new PropertyStorageSpec("lastIndexed", JdbcType.TIMESTAMP), + new PropertyStorageSpec("container", JdbcType.GUID).setNullable(false), + new PropertyStorageSpec(DataIntegrationService.Columns.TransformImportHash.getColumnName(), JdbcType.VARCHAR, 256)); + RESERVED_NAMES = DomainUtil.getNamesAndLabels( + BASE_PROPERTIES.stream().map(PropertyStorageSpec::getName).collect(Collectors.toSet())); + } + + public void setListDefinition(ListDefinitionImpl list) + { + _list = list; + } + + @Override + public String getTypeLabel(Domain domain) + { + return "List '" + domain.getName() + "'"; + } + + @Override + public SQLFragment sqlObjectIdsInDomain(Domain domain) + { + throw new UnsupportedOperationException("sqlObjectIdsInDomain NYI for ListDomainKind"); + } + + @Override + public ActionURL urlShowData(Domain domain, ContainerUser containerUser) + { + ListDefinition listDef = ListService.get().getList(domain); + if (null == listDef) + return null; + return listDef.urlShowData(); + } + + @Nullable + @Override + public ActionURL urlEditDefinition(Domain domain, ContainerUser containerUser) + { + ListDefinition listDef = ListService.get().getList(domain); + if (null == listDef) + return null; + return listDef.urlShowDefinition(); + } + + @Override + public boolean allowAttachmentProperties() + { + return true; + } + + @Override + public boolean allowMultiChoiceProperties() + { + return true; + } + + @Override + public boolean showDefaultValueSettings() + { + return true; + } + + @Override + public boolean allowUniqueConstraintProperties() + { + return true; + } + + @Override + public boolean allowCalculatedFields() + { + return true; + } + + @Override + public Priority getPriority(String domainURI) + { + Lsid lsid = new Lsid(domainURI); + return getKindName().equals(lsid.getNamespacePrefix()) ? Priority.MEDIUM : null; + } + + + @Override + public Set getBaseProperties(Domain domain) + { + Set specs = new HashSet<>(BASE_PROPERTIES); + specs.addAll(super.getBaseProperties(domain)); + return specs; + } + + @Override + public Set getMandatoryPropertyNames(Domain domain) + { + Set properties = super.getMandatoryPropertyNames(domain); + properties.addAll(getAdditionalProtectedPropertyNames(domain)); + return properties; + } + + /** + * Returns the List's primary key as a field to get special treatment elsewhere, despite being property driven. + */ + @Override + public Set getAdditionalProtectedProperties(Domain domain) + { + Set specs = new HashSet<>(super.getAdditionalProtectedProperties(domain)); + + ListDefinition listDef = ListService.get().getList(domain); + if (null != listDef) + { + String keyName = listDef.getKeyName(); + JdbcType keyType = listDef.getKeyType() == KeyType.Varchar ? JdbcType.VARCHAR : JdbcType.INTEGER; + int keySize = keyType == JdbcType.VARCHAR ? 4000 : 0; + specs.add(new PropertyStorageSpec(keyName, keyType, keySize, PropertyStorageSpec.Special.PrimaryKey)); + } + + return specs; + } + + @Override + public PropertyStorageSpec getPropertySpec(PropertyDescriptor pd, Domain domain) + { + ListDefinition list = ListService.get().getList(domain); + if (null == list) + list = _list; + + if (null != list) + { + if (pd.getName().equalsIgnoreCase(list.getKeyName())) + { + PropertyStorageSpec key = getKeyProperty(list, pd.getStorageColumnName()); + assert key.isPrimaryKey(); + _list = null; + return key; + } + } + return super.getPropertySpec(pd, domain); + } + + abstract PropertyStorageSpec getKeyProperty(ListDefinition list, String storageColumnName); + + abstract KeyType getDefaultKeyType(); + + abstract Collection getSupportedKeyTypes(); + + @Override + public @NotNull Set getReservedPropertyNames(Domain domain, User user) + { + return RESERVED_NAMES; + } + + @Override + public String getStorageSchemaName() + { + return ListSchema.getInstance().getSchemaName(); + } + + @Override + public DbScope getScope() + { + return ListSchema.getInstance().getSchema().getScope(); + } + + @Override + public Set getPropertyIndices(Domain domain) + { + // return PageFlowUtil.set(new PropertyStorageSpec.Index(false, ListDomainKind.KEY_FIELD)); + return Collections.emptySet(); // TODO: Allow this to return the Key Column + } + + public static Lsid generateDomainURI(Container c, KeyType keyType, @Nullable ListDefinition.Category category) + { + String type = getType(keyType, category); + Lsid lsid; + // assure LSID does not collide with previous lsids that may have had number names + do + { + String dbSeqStr = String.valueOf(LsidManager.getLsidPrefixDbSeq(type, 1).next()); + lsid = new Lsid(type, "Folder-" + c.getRowId(), dbSeqStr); + } while (OntologyManager.getDomainDescriptor(lsid.toString(), c) != null); + + return lsid; + } + + public static Lsid createPropertyURI(String listName, String columnName, Container c, ListDefinition.KeyType keyType) + { + StringBuilder typeURI = getBaseURI(listName, getType(keyType, null), c); + typeURI.append(".").append(PageFlowUtil.encode(columnName)); + return new Lsid(typeURI.toString()); + } + + private static String getType(KeyType keyType, @Nullable ListDefinition.Category category) + { + String type; + switch (keyType) + { + case Integer: + case AutoIncrementInteger: + if (category != null) + type = PicklistDomainKind.NAMESPACE_PREFIX; + else + type = IntegerListDomainKind.NAMESPACE_PREFIX; + break; + case Varchar: + type = VarcharListDomainKind.NAMESPACE_PREFIX; + break; + default: + throw new IllegalStateException(); + } + return type; + } + + private static StringBuilder getBaseURI(String listName, String type, Container c) + { + return new StringBuilder("urn:lsid:") + .append(PageFlowUtil.encode(AppProps.getInstance().getDefaultLsidAuthority())) + .append(":").append(type).append(".Folder-").append(c.getRowId()).append(":") + .append(PageFlowUtil.encode(listName)); + } + + @Override + public TableInfo getTableInfo(User user, Container container, String name, @Nullable ContainerFilter cf) + { + return new ListQuerySchema(user, container).createTable(name, cf); + } + + @Override + public boolean isDeleteAllDataOnFieldImport() + { + return true; + } + + @Override + public boolean canCreateDefinition(User user, Container container) + { + return container.hasPermission("ListDomainKind.canCreateDefinition", user, DesignListPermission.class); + } + + @Override + public boolean canEditDefinition(User user, Domain domain) + { + Container c = domain.getContainer(); + return c.hasPermission("ListDomainKind.canEditDefinition", user, DesignListPermission.class); + } + + @Override + public ActionURL urlCreateDefinition(String schemaName, String queryName, Container container, User user) + { + return PageFlowUtil.urlProvider(ListUrls.class).getCreateListURL(container); + } + + @Override + public Class getTypeClass() + { + return ListDomainKindProperties.class; + } + + @Override + public void validateDomainName(Container container, User user, @Nullable Domain domain, String name) + { + if (StringUtils.isEmpty(name)) + throw new ApiUsageException("List name is required."); + if (name.length() > MAX_NAME_LENGTH) + throw new ApiUsageException("List name cannot be longer than " + MAX_NAME_LENGTH + " characters."); + if (ListService.get().getList(container, name, domain == null) != null) + throw new ApiUsageException("The name '" + name + "' is already in use."); + } + + @Override + public Domain createDomain(GWTDomain domain, ListDomainKindProperties listProperties, Container container, User user, @Nullable TemplateInfo templateInfo, boolean forUpdate) + { + String name = StringUtils.trimToEmpty(domain.getName()); + validateDomainName(container, user, null, name); + + String keyName = listProperties.getKeyName(); + if (StringUtils.isEmpty(keyName)) + throw new ApiUsageException("List keyName is required"); + + KeyType keyType = getDefaultKeyType(); + + if (null != listProperties.getKeyType()) + { + String rawKeyType = listProperties.getKeyType(); + if (EnumUtils.isValidEnum(KeyType.class, rawKeyType)) + keyType = KeyType.valueOf(rawKeyType); + else + throw new ApiUsageException("List keyType provided does not exist."); + } + + ListDefinition.Category category; + try + { + category = listProperties.getCategory() != null ? ListDefinition.Category.valueOf(listProperties.getCategory()) : null; + } + catch (IllegalArgumentException e) + { + throw new ApiUsageException(String.format("List category type provided (%s) does not exist.", listProperties.getCategory())); + } + + if (!getSupportedKeyTypes().contains(keyType)) + throw new ApiUsageException("List keyType provided is not supported for list domain kind (" + getKindName() + ")."); + + ListDefinition list = ListService.get().createList(container, name, keyType, templateInfo, category); + list.setKeyName(keyName); + + // Issue 45042: Allow for the list description to be set via the create domain API calls + if (listProperties.getDescription() == null && domain.getDescription() != null) + listProperties.setDescription(domain.getDescription()); + list.setDescription(listProperties.getDescription()); + + List properties = (List)domain.getFields(); + List indices = (List)domain.getIndices(); + + try (DbScope.Transaction tx = ExperimentService.get().ensureTransaction()) + { + Domain d = list.getDomain(forUpdate); + + Set reservedNames = getReservedPropertyNames(d, user); + Set lowerReservedNames = reservedNames.stream().map(String::toLowerCase).collect(Collectors.toSet()); + + Map defaultValues = new HashMap<>(); + Set propertyUris = new CaseInsensitiveHashSet(); + for (GWTPropertyDescriptor pd : properties) + { + String propertyName = pd.getName().toLowerCase(); + if (lowerReservedNames.contains(propertyName)) + { + if (pd.getLabel() == null) + pd.setLabel(pd.getName()); + pd.setName("_" + pd.getName()); + } + + DomainUtil.addProperty(d, pd, defaultValues, propertyUris, null); + } + + d.setPropertyIndices(indices, lowerReservedNames); + + // TODO: This looks like the wrong order to me -- we should updateListProperties() (persist the indexing + // settings and handle potential transitions) before calling save() (which indexes the list). Since this is + // the create case there's no data to index, but there is meta data... + list.save(user, true, listProperties.getAuditRecordMap(), domain.getCalculatedFields()); + updateListProperties(container, user, list.getListId(), listProperties); + + QueryService.get().saveCalculatedFieldsMetadata(ListQuerySchema.NAME, name, null, domain.getCalculatedFields(), false, user, container); + + DefaultValueService.get().setDefaultValues(container, defaultValues); + + tx.commit(); + } + catch (Exception e) + { + throw new RuntimeException(e); + } + + return list.getDomain(forUpdate); + } + + @Override + public @NotNull ValidationException updateDomain(GWTDomain original, GWTDomain update, + ListDomainKindProperties listProperties, Container container, User user, boolean includeWarnings, @Nullable String auditUserComment) + { + ValidationException exception; + + try (DbScope.Transaction transaction = ListManager.get().getListMetadataSchema().getScope().ensureTransaction()) + { + exception = new ValidationException(); + StringBuilder changeDetails = new StringBuilder(); + + Domain domain = PropertyService.get().getDomain(container, original.getDomainURI()); + if (null == domain) + return exception.addGlobalError("Expected domain for list: " + original.getName()); + + ListDefinition listDefinition = ListService.get().getList(domain); + TableInfo table = listDefinition.getTable(user); + + if (null == table) + return exception.addGlobalError("Expected table for list: " + listDefinition.getName()); + + // Handle cases when existing key field is null or is not provided in the updated domainDesign + GWTPropertyDescriptor key = findField(listDefinition.getKeyName(), original.getFields()); + if (null != key) + { + int id = key.getPropertyId(); + GWTPropertyDescriptor newKey = findField(id, update.getFields()); + if (null == newKey) + { + return exception.addGlobalError("Key field not provided, expecting key field '" + key.getName() + "'"); + } + else if (!key.getName().equalsIgnoreCase(newKey.getName())) + { + return exception.addGlobalError("Cannot change key field name"); + } + } + else + { + return exception.addGlobalError("Key field not found for list '" + listDefinition.getName() + "'"); + } + + //handle name change + String updatedName = StringUtils.trim(update.getName()); + boolean hasNameChange = !original.getName().equals(updatedName); + if (hasNameChange) + { + validateDomainName(container, user, domain, updatedName); + changeDetails.append("The name of the list domain '" + original.getName() + "' was changed to '" + updatedName + "'."); + } + + //return if there are errors before moving forward with the save + if (exception.hasErrors()) + { + return exception; + } + + //update list properties + Map oldProps = null; + Map newProps = null; + if (null != listProperties) + { + if (listProperties.getDomainId() != original.getDomainId() || listProperties.getDomainId() != update.getDomainId()) + return exception.addGlobalError("domainId for the list does not match old or the new domain"); + if (!original.getDomainURI().equals(update.getDomainURI())) + return exception.addGlobalError("domainURI mismatch between old and new domain"); + + Pair, Map> updatedProps = updateListProperties(container, user, listDefinition.getListId(), listProperties); + oldProps = updatedProps.first; + newProps = updatedProps.second; + } + // Issue 45042: Allow for the list description to be set via the save domain API calls + else if (update.getDescription() != null) + { + listProperties = getListProperties(container, user, listDefinition.getListId()); + listProperties.setDescription(update.getDescription()); + Pair, Map> updatedProps = updateListProperties(container, user, listDefinition.getListId(), listProperties); + oldProps = updatedProps.first; + newProps = updatedProps.second; + } + + //update domain design properties + try + { + //handle attachment cols + Map modifiedAttachmentColumns = new CaseInsensitiveHashMap<>(); + + for (DomainProperty oldProp : domain.getProperties()) + { + if (PropertyType.ATTACHMENT.equals(oldProp.getPropertyDescriptor().getPropertyType())) + { + GWTPropertyDescriptor newGWTProp = findField(oldProp.getPropertyId(), update.getFields()); + if (null == newGWTProp || !PropertyType.ATTACHMENT.equals(PropertyType.getFromURI(newGWTProp.getConceptURI(), newGWTProp.getRangeURI(), null))) + { + ColumnInfo column = table.getColumn(oldProp.getPropertyDescriptor().getName()); + if (null != column) + modifiedAttachmentColumns.put(oldProp.getPropertyDescriptor().getName(), column); + } + } + } + + Collection> attachmentMapCollection = null; + if (!modifiedAttachmentColumns.isEmpty()) + { + List columns = new ArrayList<>(modifiedAttachmentColumns.values()); + columns.add(table.getColumn("entityId")); + attachmentMapCollection = new TableSelector(table, columns, null, null).getMapCollection(); + } + + // Remove attachments from any attachment columns that are removed or no longer attachment columns + if (null != attachmentMapCollection) + { + for (Map map : attachmentMapCollection) + { + String entityId = (String) map.get("entityId"); + ListItemAttachmentParent parent = new ListItemAttachmentParent(entityId, container); + for (Map.Entry entry : map.entrySet()) + if (null != entry.getValue() && modifiedAttachmentColumns.containsKey(entry.getKey())) + AttachmentService.get().deleteAttachment(parent, entry.getValue().toString(), user); + } + } + + //update domain properties + exception.addErrors(DomainUtil.updateDomainDescriptor(original, update, container, user, hasNameChange, changeDetails.toString(), auditUserComment, oldProps, newProps)); + + QueryService.get().saveCalculatedFieldsMetadata(ListQuerySchema.NAME, update.getQueryName(), hasNameChange ? update.getName() : null, update.getCalculatedFields(), !original.getCalculatedFields().isEmpty(), user, container); + } + catch (RuntimeSQLException x) + { + // issue 19202 - check for null value exceptions in case provided file data not contain the column + // and return a better error message + String message = x.getMessage(); + if (x.isNullValueException()) + { + message = "The provided data does not contain the specified '" + listDefinition.getKeyName() + "' field or contains null key values."; + } + return exception.addGlobalError(message); + } + catch (DataIntegrityViolationException | MetadataUnavailableException x) + { + return exception.addGlobalError("A data error occurred: " + x.getMessage()); + } + + if (!exception.hasErrors()) + { + transaction.commit(); + } + return exception; + } + } + + private ListDomainKindProperties getListProperties(Container container, User user, int listId) + { + SimpleFilter filter = SimpleFilter.createContainerFilter(container); + filter.addCondition(FieldKey.fromParts("ListId"), listId); + return new TableSelector(ListManager.get().getListMetadataTable(), filter, null).getObject(ListDomainKindProperties.class); + } + + private Pair, Map> updateListProperties(Container container, User user, int listId, ListDomainKindProperties listProperties) + { + ListDomainKindProperties existingListProps = getListProperties(container, user, listId); + + Map oldProps = existingListProps == null ? null : existingListProps.getAuditRecordMap(); + Map newProps = listProperties == null ? oldProps : listProperties.getAuditRecordMap(); + //merge existing and new properties + ListDomainKindProperties updatedListProps = updateListProperties(existingListProps, listProperties); + + ListManager.get().update(user, container, updatedListProps); + + return new Pair<>(oldProps, newProps); + } + + //updates list properties except listId, domainId, keyName, keyType, and lastIndexed + private ListDomainKindProperties updateListProperties(ListDomainKindProperties existingListProps, ListDomainKindProperties newListProps) + { + + ListDomainKindProperties updatedListProps = new ListDomainKindProperties(existingListProps); + if (null != newListProps.getName()) + updatedListProps.setName(newListProps.getName().trim()); + + updatedListProps.setTitleColumn(newListProps.getTitleColumn()); + updatedListProps.setDescription(newListProps.getDescription()); + updatedListProps.setAllowDelete(newListProps.isAllowDelete()); + updatedListProps.setAllowUpload(newListProps.isAllowUpload()); + updatedListProps.setAllowExport(newListProps.isAllowExport()); + updatedListProps.setCategory(newListProps.getCategory()); + updatedListProps.setEntireListTitleTemplate(newListProps.getEntireListTitleTemplate()); + updatedListProps.setEntireListIndexSetting(newListProps.getEntireListIndexSetting()); + updatedListProps.setEntireListBodySetting(newListProps.getEntireListBodySetting()); + updatedListProps.setEachItemTitleTemplate(newListProps.getEachItemTitleTemplate()); + updatedListProps.setEachItemBodySetting(newListProps.getEachItemBodySetting()); + updatedListProps.setEntireListIndex(newListProps.isEntireListIndex()); + updatedListProps.setEntireListBodyTemplate(newListProps.getEntireListBodyTemplate()); + updatedListProps.setEachItemIndex(newListProps.isEachItemIndex()); + updatedListProps.setEachItemBodyTemplate(newListProps.getEachItemBodyTemplate()); + updatedListProps.setFileAttachmentIndex(newListProps.isFileAttachmentIndex()); + + return updatedListProps; + } + + private GWTPropertyDescriptor findField(String name, List fields) + { + for (GWTPropertyDescriptor f : fields) + { + if (name.equalsIgnoreCase(f.getName())) + return f; + } + return null; + } + + private GWTPropertyDescriptor findField(int id, List fields) + { + if (id > 0) + { + for (GWTPropertyDescriptor f : fields) + { + if (id == f.getPropertyId()) + return f; + } + } + return null; + } + + @Override + public @Nullable ListDomainKindProperties getDomainKindProperties(GWTDomain domain, Container container, User user) + { + ListDefinition list = domain != null ? ListService.get().getList(PropertyService.get().getDomain(domain.getDomainId())) : null; + return ListManager.get().getListDomainKindProperties(container, list != null ? list.getListId() : null); + } + + @Override + public void deleteDomain(User user, Domain domain, String userComment) + { + ListDefinition list = ListService.get().getList(domain); + if (list == null) + throw new NotFoundException("List not found: " + domain.getTypeURI()); + + try + { + list.delete(user, userComment); + } + catch (DomainNotFoundException e) + { + throw new NotFoundException(e.getMessage()); + } + } + + @Override + public void invalidate(Domain domain) + { + super.invalidate(domain); + + ListDefinition list = ListService.get().getList(domain); + if (list != null) + ListManager.get().indexList(list); + } + + @Override + public boolean matchesTemplateXML(String templateName, DomainTemplateType template, List properties) + { + if (!(template instanceof ListTemplateType)) + return false; + + ListOptionsType options = ((ListTemplateType) template).getOptions(); + if (options == null) + throw new IllegalArgumentException("List template requires specifying a keyCol"); + + String keyName = options.getKeyCol(); + if (keyName == null) + throw new IllegalArgumentException("List template requires specifying a keyCol"); + + Pair pair = findProperty(templateName, properties, keyName); + GWTPropertyDescriptor prop = pair.first; + + PropertyType type = PropertyType.getFromURI(prop.getConceptURI(), prop.getRangeURI()); + + return type.equals(getDefaultKeyType().getPropertyType()); + } + + public void ensureBaseProperties(Domain d) + { + var props = getBaseProperties(d); + } + + @Override + public boolean supportsPhiLevel() + { + return ComplianceService.get().isComplianceSupported(); + } +} diff --git a/study/src/org/labkey/study/model/DatasetDomainKind.java b/study/src/org/labkey/study/model/DatasetDomainKind.java index be77b3f371d..c6b59b82ccb 100644 --- a/study/src/org/labkey/study/model/DatasetDomainKind.java +++ b/study/src/org/labkey/study/model/DatasetDomainKind.java @@ -1,866 +1,872 @@ -/* - * Copyright (c) 2008-2019 LabKey Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.labkey.study.model; - -import org.apache.commons.beanutils.BeanUtils; -import org.apache.commons.lang3.StringUtils; -import org.jetbrains.annotations.NotNull; -import org.jetbrains.annotations.Nullable; -import org.labkey.api.action.ApiUsageException; -import org.labkey.api.collections.CaseInsensitiveHashSet; -import org.labkey.api.compliance.ComplianceService; -import org.labkey.api.data.BaseColumnInfo; -import org.labkey.api.data.ColumnInfo; -import org.labkey.api.data.Container; -import org.labkey.api.data.ContainerFilter; -import org.labkey.api.data.DbScope; -import org.labkey.api.data.JdbcType; -import org.labkey.api.data.PropertyStorageSpec; -import org.labkey.api.data.RuntimeSQLException; -import org.labkey.api.data.SQLFragment; -import org.labkey.api.data.SchemaTableInfo; -import org.labkey.api.data.TableInfo; -import org.labkey.api.di.DataIntegrationService; -import org.labkey.api.exp.Lsid; -import org.labkey.api.exp.PropertyDescriptor; -import org.labkey.api.exp.TemplateInfo; -import org.labkey.api.exp.api.ExperimentService; -import org.labkey.api.exp.api.StorageProvisioner; -import org.labkey.api.exp.property.AbstractDomainKind; -import org.labkey.api.exp.property.Domain; -import org.labkey.api.exp.property.DomainProperty; -import org.labkey.api.exp.property.DomainUtil; -import org.labkey.api.exp.property.PropertyService; -import org.labkey.api.gwt.client.model.GWTDomain; -import org.labkey.api.gwt.client.model.GWTIndex; -import org.labkey.api.gwt.client.model.GWTPropertyDescriptor; -import org.labkey.api.query.MetadataUnavailableException; -import org.labkey.api.query.QueryService; -import org.labkey.api.query.ValidationException; -import org.labkey.api.reports.model.ViewCategory; -import org.labkey.api.reports.model.ViewCategoryManager; -import org.labkey.api.security.User; -import org.labkey.api.security.permissions.AdminPermission; -import org.labkey.api.study.Dataset; -import org.labkey.api.study.Dataset.KeyManagementType; -import org.labkey.api.study.Study; -import org.labkey.api.study.StudyService; -import org.labkey.api.study.TimepointType; -import org.labkey.api.util.UnexpectedException; -import org.labkey.api.view.ActionURL; -import org.labkey.api.view.NotFoundException; -import org.labkey.api.writer.ContainerUser; -import org.labkey.study.StudySchema; -import org.labkey.study.assay.StudyPublishManager; -import org.labkey.study.controllers.StudyController; -import org.labkey.study.query.StudyQuerySchema; - -import java.util.ArrayList; -import java.util.Arrays; -import java.util.Collections; -import java.util.HashMap; -import java.util.HashSet; -import java.util.List; -import java.util.Map; -import java.util.Objects; -import java.util.Set; -import java.util.concurrent.locks.Lock; -import java.util.stream.Collectors; - -import static org.labkey.api.util.IntegerUtils.asInteger; -import static org.labkey.study.model.DatasetDomainKindProperties.TIME_KEY_FIELD_KEY; - -public abstract class DatasetDomainKind extends AbstractDomainKind -{ - public final static String LSID_PREFIX = "StudyDataset"; - - public final static String CONTAINER = "container"; - public final static String DATE = "date"; - public final static String PARTICIPANTID = "participantid"; - public final static String LSID = "lsid"; - public final static String DSROWID = "dsrowid"; - public final static String SEQUENCENUM = "sequencenum"; - public final static String SOURCELSID = "sourcelsid"; - public final static String _KEY = "_key"; - public final static String QCSTATE = "qcstate"; - public final static String PARTICIPANTSEQUENCENUM = "participantsequencenum"; - - public static final String CREATED = "created"; - public static final String MODIFIED = "modified"; - public static final String CREATED_BY = "createdBy"; - public static final String MODIFIED_BY = "modifiedBy"; - - - /* - * the columns common to all datasets - */ - private final static Set BASE_PROPERTIES; - private final static Set DATASPACE_BASE_PROPERTIES; - protected final static Set PROPERTY_INDICES; - private final static Set DATASPACE_PROPERTY_INDICES; - - static - { - DATASPACE_BASE_PROPERTIES = new HashSet<>(Arrays.asList( - new PropertyStorageSpec(DSROWID, JdbcType.BIGINT, 0, PropertyStorageSpec.Special.PrimaryKeyNonClustered, false, true, null), - new PropertyStorageSpec(CONTAINER, JdbcType.GUID).setNullable(false), - new PropertyStorageSpec(PARTICIPANTID, JdbcType.VARCHAR, 32).setNullable(false), - new PropertyStorageSpec(LSID, JdbcType.VARCHAR, 200), - new PropertyStorageSpec(SEQUENCENUM, JdbcType.DECIMAL), - new PropertyStorageSpec(SOURCELSID, JdbcType.VARCHAR, 200), - new PropertyStorageSpec(_KEY, JdbcType.VARCHAR, 200), - new PropertyStorageSpec(QCSTATE, JdbcType.INTEGER), - new PropertyStorageSpec(PARTICIPANTSEQUENCENUM, JdbcType.VARCHAR, 200), - new PropertyStorageSpec(CREATED, JdbcType.TIMESTAMP), - new PropertyStorageSpec(MODIFIED, JdbcType.TIMESTAMP), - new PropertyStorageSpec(CREATED_BY, JdbcType.INTEGER), - new PropertyStorageSpec(MODIFIED_BY, JdbcType.INTEGER), - new PropertyStorageSpec(DATE, JdbcType.TIMESTAMP), - new PropertyStorageSpec(DataIntegrationService.Columns.TransformImportHash.getColumnName(), JdbcType.VARCHAR, 256) - )); - - - BASE_PROPERTIES = new HashSet<>(Arrays.asList( - new PropertyStorageSpec(DSROWID, JdbcType.BIGINT, 0, PropertyStorageSpec.Special.PrimaryKeyNonClustered, false, true, null), - new PropertyStorageSpec(PARTICIPANTID, JdbcType.VARCHAR, 32), - new PropertyStorageSpec(LSID, JdbcType.VARCHAR, 200), - new PropertyStorageSpec(SEQUENCENUM, JdbcType.DECIMAL), - new PropertyStorageSpec(SOURCELSID, JdbcType.VARCHAR, 200), - new PropertyStorageSpec(_KEY, JdbcType.VARCHAR, 200), - new PropertyStorageSpec(QCSTATE, JdbcType.INTEGER), - new PropertyStorageSpec(PARTICIPANTSEQUENCENUM, JdbcType.VARCHAR, 200), - new PropertyStorageSpec(CREATED, JdbcType.TIMESTAMP), - new PropertyStorageSpec(MODIFIED, JdbcType.TIMESTAMP), - new PropertyStorageSpec(CREATED_BY, JdbcType.INTEGER), - new PropertyStorageSpec(MODIFIED_BY, JdbcType.INTEGER), - new PropertyStorageSpec(DATE, JdbcType.TIMESTAMP), - new PropertyStorageSpec(DataIntegrationService.Columns.TransformImportHash.getColumnName(), JdbcType.VARCHAR, 256) - )); - - DATASPACE_PROPERTY_INDICES = new HashSet<>(Arrays.asList( - new PropertyStorageSpec.Index(false, true, CONTAINER, PARTICIPANTID, DATE), - new PropertyStorageSpec.Index(false, CONTAINER, QCSTATE), - new PropertyStorageSpec.Index(true, CONTAINER, PARTICIPANTID, SEQUENCENUM, _KEY), - new PropertyStorageSpec.Index(true, LSID), - new PropertyStorageSpec.Index(false, DATE) - )); - - PROPERTY_INDICES = new HashSet<>(Arrays.asList( - new PropertyStorageSpec.Index(false, true, PARTICIPANTID, DATE), - new PropertyStorageSpec.Index(false, QCSTATE), - new PropertyStorageSpec.Index(true, PARTICIPANTID, SEQUENCENUM, _KEY), - new PropertyStorageSpec.Index(true, LSID), - new PropertyStorageSpec.Index(false, DATE) - )); - } - - - protected DatasetDomainKind() - { - } - - @Override - abstract public String getKindName(); - - @Override - public Class getTypeClass() - { - return DatasetDomainKindProperties.class; - } - - @Override - public String getTypeLabel(Domain domain) - { - DatasetDefinition def = getDatasetDefinition(domain.getTypeURI()); - if (null == def) - return domain.getName(); - return def.getName(); - } - - @Override - public SQLFragment sqlObjectIdsInDomain(Domain domain) - { - DatasetDefinition def = getDatasetDefinition(domain.getTypeURI()); - if (null == def) - return new SQLFragment("NULL"); - TableInfo ti = def.getStorageTableInfo(false); - SQLFragment sql = new SQLFragment(); - sql.append("SELECT O.ObjectId FROM ").append(ti).append(" SD JOIN exp.Object O ON SD.Lsid=O.ObjectURI WHERE O.container=?"); - sql.add(def.getContainer()); - return sql; - } - - // Issue 16526: nobody should call this overload of generateDomainURI for DatasetDomainKind. Instead - // use the overload below with a unique id (the dataset's entityId). Assert is here to track down - // any callers. - // Lsid.toString() encodes incorrectly TODO: fix - @Override - public String generateDomainURI(String schemaName, String name, Container container, User user) - { - assert false; - return null; - } - - // Issue 16526: This specific generateDomainURI takes an id to uniquify the dataset. - public static String generateDomainURI(String name, String id, Container container) - { - String objectid = name == null ? "" : name; - if (null != id) - { - // normalize the object id - objectid += "-" + id.toLowerCase(); - } - return (new Lsid(LSID_PREFIX, "Folder-" + container.getRowId(), objectid)).toString(); - } - - @Override - public ActionURL urlShowData(Domain domain, ContainerUser containerUser) - { - Dataset def = getDatasetDefinition(domain.getTypeURI()); - ActionURL url = new ActionURL(StudyController.DatasetReportAction.class, containerUser.getContainer()); - url.addParameter("datasetId", "" + def.getDatasetId()); - return url; - } - - @Override - public ActionURL urlEditDefinition(Domain domain, ContainerUser containerUser) - { - Dataset def = getDatasetDefinition(domain.getTypeURI()); - ActionURL url = new ActionURL(StudyController.EditTypeAction.class, containerUser.getContainer()); - url.addParameter("datasetId", "" + def.getDatasetId()); - return url; - } - - @Override - public ActionURL urlCreateDefinition(String schemaName, String queryName, Container container, User user) - { - return new ActionURL(StudyController.DefineDatasetTypeAction.class, container); - } - - @Override - public boolean allowFileLinkProperties() - { - return true; - } - - @Override - public boolean showDefaultValueSettings() - { - return true; - } - - @Override - public boolean allowUniqueConstraintProperties() - { - return true; - } - - @Override - public boolean allowCalculatedFields() - { - return true; - } - - DatasetDefinition getDatasetDefinition(String domainURI) - { - return StudyManager.getInstance().getDatasetDefinition(domainURI); - } - - // Issue 43898: Add the study subject name column to reserved fields - protected Set getStudySubjectReservedName(Domain domain) - { - HashSet fields = new CaseInsensitiveHashSet(); - if (null != domain) - { - Study study = StudyManager.getInstance().getStudy(domain.getContainer()); - if (null != study) - { - String participantIdField = study.getSubjectColumnName(); - fields.addAll(DomainUtil.getNameAndLabels(participantIdField)); - } - } - return Collections.unmodifiableSet(fields); - } - - @Override - public abstract @NotNull Set getReservedPropertyNames(Domain domain, User user); - - @Override - public Set getBaseProperties(Domain domain) - { - Set specs; - Study study = StudyManager.getInstance().getStudy(domain.getContainer()); - - if(study == null || study.isDataspaceStudy()) - { - specs = new HashSet<>(DATASPACE_BASE_PROPERTIES); - } - else - { - specs = new HashSet<>(BASE_PROPERTIES); - } - specs.addAll(super.getBaseProperties(domain)); - return specs; - } - - @Override - public Set getPropertyIndices(Domain domain) - { - Study study = StudyManager.getInstance().getStudy(domain.getContainer()); - - if(study == null || study.isDataspaceStudy()) - return DATASPACE_PROPERTY_INDICES; - - return PROPERTY_INDICES; - } - - @Override - public DbScope getScope() - { - return StudySchema.getInstance().getSchema().getScope(); - } - - @Override - public String getStorageSchemaName() - { - return StudySchema.getInstance().getDatasetSchemaName(); - } - - @Override - public void invalidate(Domain domain) - { - super.invalidate(domain); - DatasetDefinition def = getDatasetDefinition(domain.getTypeURI()); - if (null != def) - { - StudyManager.getInstance().uncache(def); - } - } - - @Override - public boolean canCreateDefinition(User user, Container container) - { - return container.hasPermission(user, AdminPermission.class); - } - - @Override - public Map processArguments(Container container, User user, Map arguments) - { - Map updatedArguments = new HashMap<>(arguments); - - // For backwards compatibility, map "demographics" => "demographicData" - if (arguments.containsKey("demographics")) - { - updatedArguments.put("demographicData", arguments.get("demographics")); - updatedArguments.remove("demographics"); - } - - // For backwards compatibility, map "categoryId" and "categoryName" => "category" - if (arguments.containsKey("categoryId")) - { - if (arguments.containsKey("categoryName")) - throw new IllegalArgumentException("Category ID and category name cannot both be specified."); - - ViewCategory category = ViewCategoryManager.getInstance().getCategory(container, asInteger(arguments.get("categoryId"))); - if (category == null) - throw new IllegalArgumentException("Unable to find a category with the ID : " + arguments.get("categoryId") + " in this folder."); - - updatedArguments.put("category", category.getLabel()); - updatedArguments.remove("categoryId"); - } - else if (arguments.containsKey("categoryName")) - { - updatedArguments.put("category", arguments.get("categoryName")); - updatedArguments.remove("categoryName"); - } - - return updatedArguments; - } - - @Nullable - @Override - public DatasetDomainKindProperties getDomainKindProperties(GWTDomain domain, Container container, User user) - { - Dataset ds = domain != null ? getDatasetDefinition(domain.getDomainURI()) : null; - return DatasetManager.get().getDatasetDomainKindProperties(container, ds != null ? ds.getDatasetId() : null); - } - - @Override - public Domain createDomain(GWTDomain domain, DatasetDomainKindProperties arguments, Container container, User user, - @Nullable TemplateInfo templateInfo, boolean forUpdate) - { - arguments.setName(StringUtils.trimToNull(domain.getName())); - String name = arguments.getName(); - if (name == null) - throw new IllegalArgumentException("Dataset name cannot be empty."); - String description = arguments.getDescription() != null ? arguments.getDescription() : domain.getDescription(); - String label = (arguments.getLabel() == null || arguments.getLabel().isEmpty()) ? arguments.getName() : arguments.getLabel(); - Integer cohortId = arguments.getCohortId(); - String tag = arguments.getTag(); - Integer datasetId = arguments.getDatasetId(); - String categoryName = arguments.getCategory(); - boolean demographics = arguments.isDemographicData(); - boolean isManagedField = arguments.isKeyPropertyManaged(); - String visitDatePropertyName = arguments.getVisitDatePropertyName(); - boolean useTimeKeyField = arguments.isUseTimeKeyField(); - boolean showByDefault = arguments.isShowByDefault(); - String dataSharing = arguments.getDataSharing(); - - // general dataset validation - validateDatasetProperties(arguments, container, user, domain, null); - - // create-case specific validation - StudyImpl study = StudyManager.getInstance().getStudy(container); - TimepointType timepointType = study.getTimepointType(); - if (timepointType.isVisitBased() && getKindName().equals(DateDatasetDomainKind.KIND_NAME)) - throw new IllegalArgumentException("Visit based studies require a visit based dataset domain. Please specify a kind name of : " + VisitDatasetDomainKind.KIND_NAME + "."); - else if (!timepointType.isVisitBased() && getKindName().equals(VisitDatasetDomainKind.KIND_NAME)) - throw new IllegalArgumentException("Date based studies require a date based dataset domain. Please specify a kind name of : " + DateDatasetDomainKind.KIND_NAME + "."); - if (timepointType.isVisitBased() && useTimeKeyField) - throw new IllegalArgumentException("Additional key property cannot be Time (from Date/Time) for visit based studies."); - - // Check for usage of Time as Key Field - String keyPropertyName = arguments.getKeyPropertyName(); - if (useTimeKeyField) - keyPropertyName = null; - - try (DbScope.Transaction transaction = ExperimentService.get().ensureTransaction()) - { - KeyManagementType managementType = KeyManagementType.None; - if (isManagedField) - { - String rangeUri = ""; - for (GWTPropertyDescriptor a : domain.getFields()) - { - if (keyPropertyName.equalsIgnoreCase(a.getName())) - { - rangeUri = a.getRangeURI(); - break; - } - - } - - PropertyDescriptor pd = new PropertyDescriptor(); - pd.setRangeURI(rangeUri); - managementType = KeyManagementType.getManagementTypeFromProp(pd.getPropertyType()); - } - - Integer categoryId = null; - ViewCategory category; - if (categoryName != null) - { - category = ViewCategoryManager.getInstance().getCategory(container, categoryName); - if (category != null) - { - categoryId = category.getRowId(); - } - else - { - String[] parts = ViewCategoryManager.getInstance().decode(categoryName); - category = ViewCategoryManager.getInstance().ensureViewCategory(container, user, parts); - categoryId = category.getRowId(); - } - } - - DatasetDefinition def = StudyPublishManager.getInstance().createDataset(user, new DatasetDefinition.Builder(name) - .setStudy(study) - .setKeyPropertyName(keyPropertyName) - .setDatasetId(datasetId) - .setDemographicData(demographics) - .setCategoryId(categoryId) - .setUseTimeKeyField(useTimeKeyField) - .setKeyManagementType(managementType) - .setShowByDefault(showByDefault) - .setLabel(label) - .setDescription(description) - .setCohortId(cohortId) - .setTag(tag) - .setVisitDatePropertyName(visitDatePropertyName) - .setDataSharing(dataSharing)); - - if (def.getDomain() != null) - { - List properties = domain.getFields(); - - Domain newDomain = def.getDomain(true); - if (newDomain != null) - { - Set reservedNames = getReservedPropertyNames(newDomain, user); - Set lowerReservedNames = reservedNames.stream().map(String::toLowerCase).collect(Collectors.toSet()); - Set existingProperties = newDomain.getProperties().stream().map(o -> o.getName().toLowerCase()).collect(Collectors.toSet()); - Map defaultValues = new HashMap<>(); - Set propertyUris = new CaseInsensitiveHashSet(); - - for (GWTPropertyDescriptor pd : properties) - { - if (lowerReservedNames.contains(pd.getName().toLowerCase()) || existingProperties.contains(pd.getName().toLowerCase())) - { - if (arguments.isStrictFieldValidation()) - throw new ApiUsageException("Property: " + pd.getName() + " is reserved or exists in the current domain."); - } - else - DomainUtil.addProperty(newDomain, pd, defaultValues, propertyUris, null); - } - - newDomain.save(user, arguments.getAuditRecordMap(), domain.getCalculatedFields()); - - List indices = domain.getIndices(); - newDomain.setPropertyIndices(indices, lowerReservedNames); - StorageProvisioner.get().ensureTableIndices(newDomain); - - QueryService.get().saveCalculatedFieldsMetadata("study", name, null, domain.getCalculatedFields(), false, user, container); - } - else - throw new IllegalArgumentException("Failed to create domain for dataset : " + name + "."); - } - - transaction.commit(); - return study.getDataset(def.getDatasetId()).getDomain(forUpdate); - } - catch (Exception e) - { - UnexpectedException.rethrow(e); // don't re-wrap runtime exceptions - return null; // can't get here - } - } - - private void validateDatasetProperties(DatasetDomainKindProperties datasetProperties, Container container, User user, GWTDomain domain, DatasetDefinition def) - { - String name = StringUtils.trimToEmpty(datasetProperties.getName()); - String keyPropertyName = datasetProperties.getKeyPropertyName(); - Integer datasetId = datasetProperties.getDatasetId(); - boolean isManagedField = datasetProperties.isKeyPropertyManaged(); - boolean useTimeKeyField = datasetProperties.isUseTimeKeyField(); - boolean isDemographicData = datasetProperties.isDemographicData(); - - if (!container.hasPermission(user, AdminPermission.class)) - throw new IllegalArgumentException("You do not have permissions to edit dataset definitions in this container."); - - Study study = StudyService.get().getStudy(container); - if (study == null) - throw new IllegalArgumentException("A study does not exist for this container."); - - // Name related exceptions - - if (name == null || name.isEmpty()) - throw new IllegalArgumentException("Dataset name cannot be empty."); - - if (name.length() > 200) - throw new IllegalArgumentException("Dataset name must be under 200 characters."); - - if (!name.equals(domain.getName()) || def == null) - { - // issue 17766: check if dataset or query exist with this name - if (null != StudyManager.getInstance().getDatasetDefinitionByName(study, name) || null != QueryService.get().getQueryDef(user, container, "study", name)) - throw new IllegalArgumentException("A Dataset or Query already exists with the name \"" + name + "\"."); - - StudyQuerySchema schema = StudyQuerySchema.createSchema(StudyManager.getInstance().getStudy(container), User.getSearchUser()); - - // Now check standard study tables - if (schema.getTableNames().contains(name)) - throw new IllegalArgumentException("A study table exists with the name \"" + name + "\"."); - - String datasetNameError = DomainUtil.validateDomainName(name, getKindName(), false); - if (datasetNameError != null) - throw new IllegalArgumentException(datasetNameError); - } - - // Label related exceptions - - String label = Objects.requireNonNullElse(datasetProperties.getLabel(), name); // Need to verify label uniqueness even if label is unspecified - - if ((def == null || !def.getLabel().equals(label)) && null != StudyManager.getInstance().getDatasetDefinitionByLabel(study, label)) - throw new IllegalArgumentException("A Dataset already exists with the label \"" + label +"\"."); - - // Additional key related exceptions - - if ("".equals(keyPropertyName)) - throw new IllegalArgumentException("Please select a field name for the additional key."); - - if (isManagedField && (keyPropertyName == null || keyPropertyName.isEmpty())) - throw new IllegalArgumentException("Additional Key Column name must be specified if field is managed."); - - if (useTimeKeyField && isManagedField) - throw new IllegalArgumentException("Additional key cannot be a managed field if KeyPropertyName is Time (from Date/Time)."); - - if (useTimeKeyField && !(keyPropertyName == null || keyPropertyName.equals(TIME_KEY_FIELD_KEY))) - throw new IllegalArgumentException("KeyPropertyName should not be provided when using additional key of Time (from Date/Time)."); - - if (isDemographicData && (isManagedField || keyPropertyName != null)) - throw new IllegalArgumentException("There cannot be an Additional Key Column if the dataset is Demographic Data."); - - if (!useTimeKeyField && null != keyPropertyName && null == domain.getFieldByName(keyPropertyName)) - throw new IllegalArgumentException("Additional Key Column name \"" + keyPropertyName +"\" must be the name of a column."); - - if (null != keyPropertyName && isManagedField) - { - String rangeURI = domain.getFieldByName(keyPropertyName).getRangeURI(); - if (!(rangeURI.endsWith("int") || rangeURI.endsWith("double") || rangeURI.endsWith("string"))) - throw new IllegalArgumentException("If Additional Key Column is managed, the column type must be numeric or text-based."); - } - - // Other exception(s) - - if (null != datasetId && (null == def || def.getDatasetId() != datasetId) && null != study.getDataset(datasetId)) - throw new IllegalArgumentException("A Dataset already exists with the datasetId \"" + datasetId +"\"."); - - if (!study.getShareVisitDefinitions() && null != datasetProperties.getDataSharing() && !datasetProperties.getDataSharing().equals("NONE")) - throw new IllegalArgumentException("Illegal value set for data sharing option."); - } - - private void checkCanUpdate(DatasetDefinition def, Container container, User user, DatasetDomainKindProperties datasetProperties, - GWTDomain original, GWTDomain update) - { - if (null == def) - throw new IllegalArgumentException("Dataset not found."); - - if (!def.canUpdateDefinition(user)) - throw new IllegalArgumentException("Shared dataset can not be edited in this folder."); - - if (datasetProperties.getLabel() == null || datasetProperties.getLabel().isEmpty()) - throw new IllegalArgumentException("Dataset label cannot be empty."); - - if (null == PropertyService.get().getDomain(container, update.getDomainURI())) - throw new IllegalArgumentException("Domain not found: " + update.getDomainURI() + "."); - - if (!def.getTypeURI().equals(original.getDomainURI()) || !def.getTypeURI().equals(update.getDomainURI())) - throw new IllegalArgumentException("Illegal Argument"); - - if (datasetProperties.isDemographicData() && !def.isDemographicData() && !StudyManager.getInstance().isDataUniquePerParticipant(def)) - { - String noun = StudyService.get().getSubjectNounSingular(container); - throw new IllegalArgumentException("This dataset currently contains more than one row of data per " + noun + - ". Demographic data includes one row of data per " + noun + "."); - } - } - - private @NotNull ValidationException updateDomainDescriptor(GWTDomain original, GWTDomain update, - @Nullable DatasetDefinition oldDef, @Nullable DatasetDomainKindProperties datasetPropertiesUpdate, Container container, User user, String userComment) - { - StringBuilder changeDetails = new StringBuilder(); - boolean hasNameChange = false; - Map oldProps = null; - Map newProps = datasetPropertiesUpdate != null ? datasetPropertiesUpdate.getAuditRecordMap() : null; - - if (oldDef != null) - { - oldProps = oldDef.getAuditRecordMap(); - if (newProps == null) - newProps = oldProps; // no update - - hasNameChange = !datasetPropertiesUpdate.getName().equals(oldDef.getName()); - if (hasNameChange) - changeDetails.append("The name of the dataset '" + oldDef.getName() + "' was changed to '" + datasetPropertiesUpdate.getName() + "'."); - } - - ValidationException exception = new ValidationException(); - exception.addErrors(DomainUtil.updateDomainDescriptor(original, update, container, user, hasNameChange, changeDetails.toString(), userComment, oldProps, newProps)); - return exception; - } - - private ValidationException updateDataset(DatasetDomainKindProperties datasetProperties, String domainURI, ValidationException exception, - StudyImpl study, Container container, User user, DatasetDefinition def) - { - try - { - // Check for usage of Time as Key Field - boolean useTimeKeyField = datasetProperties.isUseTimeKeyField(); - if (useTimeKeyField) - datasetProperties.setKeyPropertyName(null); - - // Default is no key management - KeyManagementType keyType = KeyManagementType.None; - String keyPropertyName = datasetProperties.getKeyPropertyName(); - if (datasetProperties.getKeyPropertyName() != null) - { - Domain domain = PropertyService.get().getDomain(container, domainURI); - for (DomainProperty dp : domain.getProperties()) - { - if (dp.getName().equalsIgnoreCase(datasetProperties.getKeyPropertyName())) - { - keyPropertyName = dp.getName(); - - // Be sure that the user really wants a managed key, not just that disabled select box still had a value - if (datasetProperties.isKeyPropertyManaged()) - keyType = KeyManagementType.getManagementTypeFromProp(dp.getPropertyDescriptor().getPropertyType()); - - break; - } - } - } - - DatasetDefinition updated = def.createMutable(); - - // Clear the category ID so that it gets regenerated based on the new string - see issue 19649 - updated.setCategoryId(null); - - BeanUtils.copyProperties(updated, datasetProperties); - updated.setKeyPropertyName(keyPropertyName); - updated.setKeyManagementType(keyType); - updated.setUseTimeKeyField(useTimeKeyField); - - List errors = new ArrayList<>(); - StudyManager.getInstance().updateDatasetDefinition(user, updated, errors); - StudyManager.datasetModified(updated, true); - - for (String errorMsg: errors) - exception.addGlobalError(errorMsg); - return exception; - } - catch (RuntimeSQLException e) - { - return exception.addGlobalError("Additional key column must have unique values."); - } - catch (Exception e) - { - throw new RuntimeException(e); - } - } - - @Override - public @NotNull ValidationException updateDomain(GWTDomain original, GWTDomain update, - @Nullable DatasetDomainKindProperties datasetProperties, Container container, User user, boolean includeWarnings, String userComment) - { - assert original.getDomainURI().equals(update.getDomainURI()); - StudyImpl study = StudyManager.getInstance().getStudy(container); - DatasetDefinition def = null; - boolean hasNameChange = false; - - if (datasetProperties != null) - { - def = study.getDataset(datasetProperties.getDatasetId()); - validateDatasetProperties(datasetProperties, container, user, update, def); - checkCanUpdate(def, container, user, datasetProperties, original, update); - String updatedName = StringUtils.trimToEmpty(datasetProperties.getName()); - datasetProperties.setName(updatedName); - hasNameChange = !def.getName().equals(updatedName); - } - - // Acquire lock before we actually start the transaction to avoid deadlocks when it's refreshed during the process - Lock[] locks = def == null ? new Lock[0] : new Lock[] { def.getDomainLoadingLock() }; - try (DbScope.Transaction transaction = StudySchema.getInstance().getScope().ensureTransaction(locks)) - { - ValidationException exception = updateDomainDescriptor(original, update, def, datasetProperties, container, user, userComment); - - QueryService.get().saveCalculatedFieldsMetadata("study", update.getQueryName(), hasNameChange ? datasetProperties.getName() : null, update.getCalculatedFields(), !original.getCalculatedFields().isEmpty(), user, container); - - if (!exception.hasErrors() && def != null) - exception = updateDataset(datasetProperties, original.getDomainURI(), exception, study, container, user, def); - - if (!exception.hasErrors()) - transaction.commit(); - - return exception; - } - catch (MetadataUnavailableException e) - { - return new ValidationException(e.getMessage()); - } - finally - { - if (def != null) - StudyManager.getInstance().uncache(def); - } - } - - @Override - public void deleteDomain(User user, Domain domain, @Nullable String auditUserComment) - { - DatasetDefinition def = StudyManager.getInstance().getDatasetDefinition(domain.getTypeURI()); - if (def == null) - throw new NotFoundException("Dataset not found: " + domain.getTypeURI()); - - StudyImpl study = StudyManager.getInstance().getStudy(domain.getContainer()); - if (study == null) - throw new IllegalArgumentException("A study does not exist for this folder"); - - try (DbScope.Transaction transaction = StudySchema.getInstance().getSchema().getScope().ensureTransaction()) - { - StudyManager.getInstance().deleteDataset(study, user, def, false, auditUserComment); - transaction.commit(); - } - } - - @Override - public boolean isDeleteAllDataOnFieldImport() - { - return true; - } - - - @Override - public TableInfo getTableInfo(User user, Container container, String name, @Nullable ContainerFilter cf) - { - StudyImpl study = StudyManager.getInstance().getStudy(container); - if (null == study) - return null; - StudyQuerySchema schema = StudyQuerySchema.createSchema(study, user); - DatasetDefinition dsd = schema.getDatasetDefinitionByName(name); - if (null == dsd) - return null; - - return schema.getTable(name, cf, true, false); - } - - @Override - public void afterLoadTable(SchemaTableInfo ti, Domain domain) - { - // Grab the "standard" properties and apply them to this dataset table - TableInfo template = DatasetDefinition.getTemplateTableInfo(); - - for (PropertyStorageSpec pss : domain.getDomainKind().getBaseProperties(domain)) - { - ColumnInfo c = ti.getColumn(pss.getName()); - ColumnInfo tCol = template.getColumn(pss.getName()); - // The column may be null if the dataset is being deleted in the background - if (null != tCol && c != null) - { - ((BaseColumnInfo)c).setExtraAttributesFrom(tCol); - - // When copying a column, the hidden bit is not propagated, so we need to do it manually - if (tCol.isHidden()) - ((BaseColumnInfo)c).setHidden(true); - } - } - } - - @Override - public boolean supportsPhiLevel() - { - return ComplianceService.get().isComplianceSupported(); - } - - // Query datasets don't have a domain kind, so check here if the dataset is a query dataset - @Override - public boolean isProvisioned(Container container, String name) - { - return super.isProvisioned(container, name) && !isQueryDataset(container, name); - } - - public static boolean isQueryDataset(Container container, String queryName) - { - StudyService ss = StudyService.get(); - if (ss == null) - return false; - - Dataset dt = ss.getDataset(container, ss.getDatasetIdByName(container, queryName)); - if (dt == null) - return false; - - return dt.isQueryDataset(); - } -} +/* + * Copyright (c) 2008-2019 LabKey Corporation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.labkey.study.model; + +import org.apache.commons.beanutils.BeanUtils; +import org.apache.commons.lang3.StringUtils; +import org.jetbrains.annotations.NotNull; +import org.jetbrains.annotations.Nullable; +import org.labkey.api.action.ApiUsageException; +import org.labkey.api.collections.CaseInsensitiveHashSet; +import org.labkey.api.compliance.ComplianceService; +import org.labkey.api.data.BaseColumnInfo; +import org.labkey.api.data.ColumnInfo; +import org.labkey.api.data.Container; +import org.labkey.api.data.ContainerFilter; +import org.labkey.api.data.DbScope; +import org.labkey.api.data.JdbcType; +import org.labkey.api.data.PropertyStorageSpec; +import org.labkey.api.data.RuntimeSQLException; +import org.labkey.api.data.SQLFragment; +import org.labkey.api.data.SchemaTableInfo; +import org.labkey.api.data.TableInfo; +import org.labkey.api.di.DataIntegrationService; +import org.labkey.api.exp.Lsid; +import org.labkey.api.exp.PropertyDescriptor; +import org.labkey.api.exp.TemplateInfo; +import org.labkey.api.exp.api.ExperimentService; +import org.labkey.api.exp.api.StorageProvisioner; +import org.labkey.api.exp.property.AbstractDomainKind; +import org.labkey.api.exp.property.Domain; +import org.labkey.api.exp.property.DomainProperty; +import org.labkey.api.exp.property.DomainUtil; +import org.labkey.api.exp.property.PropertyService; +import org.labkey.api.gwt.client.model.GWTDomain; +import org.labkey.api.gwt.client.model.GWTIndex; +import org.labkey.api.gwt.client.model.GWTPropertyDescriptor; +import org.labkey.api.query.MetadataUnavailableException; +import org.labkey.api.query.QueryService; +import org.labkey.api.query.ValidationException; +import org.labkey.api.reports.model.ViewCategory; +import org.labkey.api.reports.model.ViewCategoryManager; +import org.labkey.api.security.User; +import org.labkey.api.security.permissions.AdminPermission; +import org.labkey.api.study.Dataset; +import org.labkey.api.study.Dataset.KeyManagementType; +import org.labkey.api.study.Study; +import org.labkey.api.study.StudyService; +import org.labkey.api.study.TimepointType; +import org.labkey.api.util.UnexpectedException; +import org.labkey.api.view.ActionURL; +import org.labkey.api.view.NotFoundException; +import org.labkey.api.writer.ContainerUser; +import org.labkey.study.StudySchema; +import org.labkey.study.assay.StudyPublishManager; +import org.labkey.study.controllers.StudyController; +import org.labkey.study.query.StudyQuerySchema; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.Set; +import java.util.concurrent.locks.Lock; +import java.util.stream.Collectors; + +import static org.labkey.api.util.IntegerUtils.asInteger; +import static org.labkey.study.model.DatasetDomainKindProperties.TIME_KEY_FIELD_KEY; + +public abstract class DatasetDomainKind extends AbstractDomainKind +{ + public final static String LSID_PREFIX = "StudyDataset"; + + public final static String CONTAINER = "container"; + public final static String DATE = "date"; + public final static String PARTICIPANTID = "participantid"; + public final static String LSID = "lsid"; + public final static String DSROWID = "dsrowid"; + public final static String SEQUENCENUM = "sequencenum"; + public final static String SOURCELSID = "sourcelsid"; + public final static String _KEY = "_key"; + public final static String QCSTATE = "qcstate"; + public final static String PARTICIPANTSEQUENCENUM = "participantsequencenum"; + + public static final String CREATED = "created"; + public static final String MODIFIED = "modified"; + public static final String CREATED_BY = "createdBy"; + public static final String MODIFIED_BY = "modifiedBy"; + + + /* + * the columns common to all datasets + */ + private final static Set BASE_PROPERTIES; + private final static Set DATASPACE_BASE_PROPERTIES; + protected final static Set PROPERTY_INDICES; + private final static Set DATASPACE_PROPERTY_INDICES; + + static + { + DATASPACE_BASE_PROPERTIES = new HashSet<>(Arrays.asList( + new PropertyStorageSpec(DSROWID, JdbcType.BIGINT, 0, PropertyStorageSpec.Special.PrimaryKeyNonClustered, false, true, null), + new PropertyStorageSpec(CONTAINER, JdbcType.GUID).setNullable(false), + new PropertyStorageSpec(PARTICIPANTID, JdbcType.VARCHAR, 32).setNullable(false), + new PropertyStorageSpec(LSID, JdbcType.VARCHAR, 200), + new PropertyStorageSpec(SEQUENCENUM, JdbcType.DECIMAL), + new PropertyStorageSpec(SOURCELSID, JdbcType.VARCHAR, 200), + new PropertyStorageSpec(_KEY, JdbcType.VARCHAR, 200), + new PropertyStorageSpec(QCSTATE, JdbcType.INTEGER), + new PropertyStorageSpec(PARTICIPANTSEQUENCENUM, JdbcType.VARCHAR, 200), + new PropertyStorageSpec(CREATED, JdbcType.TIMESTAMP), + new PropertyStorageSpec(MODIFIED, JdbcType.TIMESTAMP), + new PropertyStorageSpec(CREATED_BY, JdbcType.INTEGER), + new PropertyStorageSpec(MODIFIED_BY, JdbcType.INTEGER), + new PropertyStorageSpec(DATE, JdbcType.TIMESTAMP), + new PropertyStorageSpec(DataIntegrationService.Columns.TransformImportHash.getColumnName(), JdbcType.VARCHAR, 256) + )); + + + BASE_PROPERTIES = new HashSet<>(Arrays.asList( + new PropertyStorageSpec(DSROWID, JdbcType.BIGINT, 0, PropertyStorageSpec.Special.PrimaryKeyNonClustered, false, true, null), + new PropertyStorageSpec(PARTICIPANTID, JdbcType.VARCHAR, 32), + new PropertyStorageSpec(LSID, JdbcType.VARCHAR, 200), + new PropertyStorageSpec(SEQUENCENUM, JdbcType.DECIMAL), + new PropertyStorageSpec(SOURCELSID, JdbcType.VARCHAR, 200), + new PropertyStorageSpec(_KEY, JdbcType.VARCHAR, 200), + new PropertyStorageSpec(QCSTATE, JdbcType.INTEGER), + new PropertyStorageSpec(PARTICIPANTSEQUENCENUM, JdbcType.VARCHAR, 200), + new PropertyStorageSpec(CREATED, JdbcType.TIMESTAMP), + new PropertyStorageSpec(MODIFIED, JdbcType.TIMESTAMP), + new PropertyStorageSpec(CREATED_BY, JdbcType.INTEGER), + new PropertyStorageSpec(MODIFIED_BY, JdbcType.INTEGER), + new PropertyStorageSpec(DATE, JdbcType.TIMESTAMP), + new PropertyStorageSpec(DataIntegrationService.Columns.TransformImportHash.getColumnName(), JdbcType.VARCHAR, 256) + )); + + DATASPACE_PROPERTY_INDICES = new HashSet<>(Arrays.asList( + new PropertyStorageSpec.Index(false, true, CONTAINER, PARTICIPANTID, DATE), + new PropertyStorageSpec.Index(false, CONTAINER, QCSTATE), + new PropertyStorageSpec.Index(true, CONTAINER, PARTICIPANTID, SEQUENCENUM, _KEY), + new PropertyStorageSpec.Index(true, LSID), + new PropertyStorageSpec.Index(false, DATE) + )); + + PROPERTY_INDICES = new HashSet<>(Arrays.asList( + new PropertyStorageSpec.Index(false, true, PARTICIPANTID, DATE), + new PropertyStorageSpec.Index(false, QCSTATE), + new PropertyStorageSpec.Index(true, PARTICIPANTID, SEQUENCENUM, _KEY), + new PropertyStorageSpec.Index(true, LSID), + new PropertyStorageSpec.Index(false, DATE) + )); + } + + + protected DatasetDomainKind() + { + } + + @Override + abstract public String getKindName(); + + @Override + public Class getTypeClass() + { + return DatasetDomainKindProperties.class; + } + + @Override + public String getTypeLabel(Domain domain) + { + DatasetDefinition def = getDatasetDefinition(domain.getTypeURI()); + if (null == def) + return domain.getName(); + return def.getName(); + } + + @Override + public SQLFragment sqlObjectIdsInDomain(Domain domain) + { + DatasetDefinition def = getDatasetDefinition(domain.getTypeURI()); + if (null == def) + return new SQLFragment("NULL"); + TableInfo ti = def.getStorageTableInfo(false); + SQLFragment sql = new SQLFragment(); + sql.append("SELECT O.ObjectId FROM ").append(ti).append(" SD JOIN exp.Object O ON SD.Lsid=O.ObjectURI WHERE O.container=?"); + sql.add(def.getContainer()); + return sql; + } + + // Issue 16526: nobody should call this overload of generateDomainURI for DatasetDomainKind. Instead + // use the overload below with a unique id (the dataset's entityId). Assert is here to track down + // any callers. + // Lsid.toString() encodes incorrectly TODO: fix + @Override + public String generateDomainURI(String schemaName, String name, Container container, User user) + { + assert false; + return null; + } + + // Issue 16526: This specific generateDomainURI takes an id to uniquify the dataset. + public static String generateDomainURI(String name, String id, Container container) + { + String objectid = name == null ? "" : name; + if (null != id) + { + // normalize the object id + objectid += "-" + id.toLowerCase(); + } + return (new Lsid(LSID_PREFIX, "Folder-" + container.getRowId(), objectid)).toString(); + } + + @Override + public ActionURL urlShowData(Domain domain, ContainerUser containerUser) + { + Dataset def = getDatasetDefinition(domain.getTypeURI()); + ActionURL url = new ActionURL(StudyController.DatasetReportAction.class, containerUser.getContainer()); + url.addParameter("datasetId", "" + def.getDatasetId()); + return url; + } + + @Override + public ActionURL urlEditDefinition(Domain domain, ContainerUser containerUser) + { + Dataset def = getDatasetDefinition(domain.getTypeURI()); + ActionURL url = new ActionURL(StudyController.EditTypeAction.class, containerUser.getContainer()); + url.addParameter("datasetId", "" + def.getDatasetId()); + return url; + } + + @Override + public ActionURL urlCreateDefinition(String schemaName, String queryName, Container container, User user) + { + return new ActionURL(StudyController.DefineDatasetTypeAction.class, container); + } + + @Override + public boolean allowFileLinkProperties() + { + return true; + } + + @Override + public boolean allowMultiChoiceProperties() + { + return true; + } + + @Override + public boolean showDefaultValueSettings() + { + return true; + } + + @Override + public boolean allowUniqueConstraintProperties() + { + return true; + } + + @Override + public boolean allowCalculatedFields() + { + return true; + } + + DatasetDefinition getDatasetDefinition(String domainURI) + { + return StudyManager.getInstance().getDatasetDefinition(domainURI); + } + + // Issue 43898: Add the study subject name column to reserved fields + protected Set getStudySubjectReservedName(Domain domain) + { + HashSet fields = new CaseInsensitiveHashSet(); + if (null != domain) + { + Study study = StudyManager.getInstance().getStudy(domain.getContainer()); + if (null != study) + { + String participantIdField = study.getSubjectColumnName(); + fields.addAll(DomainUtil.getNameAndLabels(participantIdField)); + } + } + return Collections.unmodifiableSet(fields); + } + + @Override + public abstract @NotNull Set getReservedPropertyNames(Domain domain, User user); + + @Override + public Set getBaseProperties(Domain domain) + { + Set specs; + Study study = StudyManager.getInstance().getStudy(domain.getContainer()); + + if(study == null || study.isDataspaceStudy()) + { + specs = new HashSet<>(DATASPACE_BASE_PROPERTIES); + } + else + { + specs = new HashSet<>(BASE_PROPERTIES); + } + specs.addAll(super.getBaseProperties(domain)); + return specs; + } + + @Override + public Set getPropertyIndices(Domain domain) + { + Study study = StudyManager.getInstance().getStudy(domain.getContainer()); + + if(study == null || study.isDataspaceStudy()) + return DATASPACE_PROPERTY_INDICES; + + return PROPERTY_INDICES; + } + + @Override + public DbScope getScope() + { + return StudySchema.getInstance().getSchema().getScope(); + } + + @Override + public String getStorageSchemaName() + { + return StudySchema.getInstance().getDatasetSchemaName(); + } + + @Override + public void invalidate(Domain domain) + { + super.invalidate(domain); + DatasetDefinition def = getDatasetDefinition(domain.getTypeURI()); + if (null != def) + { + StudyManager.getInstance().uncache(def); + } + } + + @Override + public boolean canCreateDefinition(User user, Container container) + { + return container.hasPermission(user, AdminPermission.class); + } + + @Override + public Map processArguments(Container container, User user, Map arguments) + { + Map updatedArguments = new HashMap<>(arguments); + + // For backwards compatibility, map "demographics" => "demographicData" + if (arguments.containsKey("demographics")) + { + updatedArguments.put("demographicData", arguments.get("demographics")); + updatedArguments.remove("demographics"); + } + + // For backwards compatibility, map "categoryId" and "categoryName" => "category" + if (arguments.containsKey("categoryId")) + { + if (arguments.containsKey("categoryName")) + throw new IllegalArgumentException("Category ID and category name cannot both be specified."); + + ViewCategory category = ViewCategoryManager.getInstance().getCategory(container, asInteger(arguments.get("categoryId"))); + if (category == null) + throw new IllegalArgumentException("Unable to find a category with the ID : " + arguments.get("categoryId") + " in this folder."); + + updatedArguments.put("category", category.getLabel()); + updatedArguments.remove("categoryId"); + } + else if (arguments.containsKey("categoryName")) + { + updatedArguments.put("category", arguments.get("categoryName")); + updatedArguments.remove("categoryName"); + } + + return updatedArguments; + } + + @Nullable + @Override + public DatasetDomainKindProperties getDomainKindProperties(GWTDomain domain, Container container, User user) + { + Dataset ds = domain != null ? getDatasetDefinition(domain.getDomainURI()) : null; + return DatasetManager.get().getDatasetDomainKindProperties(container, ds != null ? ds.getDatasetId() : null); + } + + @Override + public Domain createDomain(GWTDomain domain, DatasetDomainKindProperties arguments, Container container, User user, + @Nullable TemplateInfo templateInfo, boolean forUpdate) + { + arguments.setName(StringUtils.trimToNull(domain.getName())); + String name = arguments.getName(); + if (name == null) + throw new IllegalArgumentException("Dataset name cannot be empty."); + String description = arguments.getDescription() != null ? arguments.getDescription() : domain.getDescription(); + String label = (arguments.getLabel() == null || arguments.getLabel().isEmpty()) ? arguments.getName() : arguments.getLabel(); + Integer cohortId = arguments.getCohortId(); + String tag = arguments.getTag(); + Integer datasetId = arguments.getDatasetId(); + String categoryName = arguments.getCategory(); + boolean demographics = arguments.isDemographicData(); + boolean isManagedField = arguments.isKeyPropertyManaged(); + String visitDatePropertyName = arguments.getVisitDatePropertyName(); + boolean useTimeKeyField = arguments.isUseTimeKeyField(); + boolean showByDefault = arguments.isShowByDefault(); + String dataSharing = arguments.getDataSharing(); + + // general dataset validation + validateDatasetProperties(arguments, container, user, domain, null); + + // create-case specific validation + StudyImpl study = StudyManager.getInstance().getStudy(container); + TimepointType timepointType = study.getTimepointType(); + if (timepointType.isVisitBased() && getKindName().equals(DateDatasetDomainKind.KIND_NAME)) + throw new IllegalArgumentException("Visit based studies require a visit based dataset domain. Please specify a kind name of : " + VisitDatasetDomainKind.KIND_NAME + "."); + else if (!timepointType.isVisitBased() && getKindName().equals(VisitDatasetDomainKind.KIND_NAME)) + throw new IllegalArgumentException("Date based studies require a date based dataset domain. Please specify a kind name of : " + DateDatasetDomainKind.KIND_NAME + "."); + if (timepointType.isVisitBased() && useTimeKeyField) + throw new IllegalArgumentException("Additional key property cannot be Time (from Date/Time) for visit based studies."); + + // Check for usage of Time as Key Field + String keyPropertyName = arguments.getKeyPropertyName(); + if (useTimeKeyField) + keyPropertyName = null; + + try (DbScope.Transaction transaction = ExperimentService.get().ensureTransaction()) + { + KeyManagementType managementType = KeyManagementType.None; + if (isManagedField) + { + String rangeUri = ""; + for (GWTPropertyDescriptor a : domain.getFields()) + { + if (keyPropertyName.equalsIgnoreCase(a.getName())) + { + rangeUri = a.getRangeURI(); + break; + } + + } + + PropertyDescriptor pd = new PropertyDescriptor(); + pd.setRangeURI(rangeUri); + managementType = KeyManagementType.getManagementTypeFromProp(pd.getPropertyType()); + } + + Integer categoryId = null; + ViewCategory category; + if (categoryName != null) + { + category = ViewCategoryManager.getInstance().getCategory(container, categoryName); + if (category != null) + { + categoryId = category.getRowId(); + } + else + { + String[] parts = ViewCategoryManager.getInstance().decode(categoryName); + category = ViewCategoryManager.getInstance().ensureViewCategory(container, user, parts); + categoryId = category.getRowId(); + } + } + + DatasetDefinition def = StudyPublishManager.getInstance().createDataset(user, new DatasetDefinition.Builder(name) + .setStudy(study) + .setKeyPropertyName(keyPropertyName) + .setDatasetId(datasetId) + .setDemographicData(demographics) + .setCategoryId(categoryId) + .setUseTimeKeyField(useTimeKeyField) + .setKeyManagementType(managementType) + .setShowByDefault(showByDefault) + .setLabel(label) + .setDescription(description) + .setCohortId(cohortId) + .setTag(tag) + .setVisitDatePropertyName(visitDatePropertyName) + .setDataSharing(dataSharing)); + + if (def.getDomain() != null) + { + List properties = domain.getFields(); + + Domain newDomain = def.getDomain(true); + if (newDomain != null) + { + Set reservedNames = getReservedPropertyNames(newDomain, user); + Set lowerReservedNames = reservedNames.stream().map(String::toLowerCase).collect(Collectors.toSet()); + Set existingProperties = newDomain.getProperties().stream().map(o -> o.getName().toLowerCase()).collect(Collectors.toSet()); + Map defaultValues = new HashMap<>(); + Set propertyUris = new CaseInsensitiveHashSet(); + + for (GWTPropertyDescriptor pd : properties) + { + if (lowerReservedNames.contains(pd.getName().toLowerCase()) || existingProperties.contains(pd.getName().toLowerCase())) + { + if (arguments.isStrictFieldValidation()) + throw new ApiUsageException("Property: " + pd.getName() + " is reserved or exists in the current domain."); + } + else + DomainUtil.addProperty(newDomain, pd, defaultValues, propertyUris, null); + } + + newDomain.save(user, arguments.getAuditRecordMap(), domain.getCalculatedFields()); + + List indices = domain.getIndices(); + newDomain.setPropertyIndices(indices, lowerReservedNames); + StorageProvisioner.get().ensureTableIndices(newDomain); + + QueryService.get().saveCalculatedFieldsMetadata("study", name, null, domain.getCalculatedFields(), false, user, container); + } + else + throw new IllegalArgumentException("Failed to create domain for dataset : " + name + "."); + } + + transaction.commit(); + return study.getDataset(def.getDatasetId()).getDomain(forUpdate); + } + catch (Exception e) + { + UnexpectedException.rethrow(e); // don't re-wrap runtime exceptions + return null; // can't get here + } + } + + private void validateDatasetProperties(DatasetDomainKindProperties datasetProperties, Container container, User user, GWTDomain domain, DatasetDefinition def) + { + String name = StringUtils.trimToEmpty(datasetProperties.getName()); + String keyPropertyName = datasetProperties.getKeyPropertyName(); + Integer datasetId = datasetProperties.getDatasetId(); + boolean isManagedField = datasetProperties.isKeyPropertyManaged(); + boolean useTimeKeyField = datasetProperties.isUseTimeKeyField(); + boolean isDemographicData = datasetProperties.isDemographicData(); + + if (!container.hasPermission(user, AdminPermission.class)) + throw new IllegalArgumentException("You do not have permissions to edit dataset definitions in this container."); + + Study study = StudyService.get().getStudy(container); + if (study == null) + throw new IllegalArgumentException("A study does not exist for this container."); + + // Name related exceptions + + if (name == null || name.isEmpty()) + throw new IllegalArgumentException("Dataset name cannot be empty."); + + if (name.length() > 200) + throw new IllegalArgumentException("Dataset name must be under 200 characters."); + + if (!name.equals(domain.getName()) || def == null) + { + // issue 17766: check if dataset or query exist with this name + if (null != StudyManager.getInstance().getDatasetDefinitionByName(study, name) || null != QueryService.get().getQueryDef(user, container, "study", name)) + throw new IllegalArgumentException("A Dataset or Query already exists with the name \"" + name + "\"."); + + StudyQuerySchema schema = StudyQuerySchema.createSchema(StudyManager.getInstance().getStudy(container), User.getSearchUser()); + + // Now check standard study tables + if (schema.getTableNames().contains(name)) + throw new IllegalArgumentException("A study table exists with the name \"" + name + "\"."); + + String datasetNameError = DomainUtil.validateDomainName(name, getKindName(), false); + if (datasetNameError != null) + throw new IllegalArgumentException(datasetNameError); + } + + // Label related exceptions + + String label = Objects.requireNonNullElse(datasetProperties.getLabel(), name); // Need to verify label uniqueness even if label is unspecified + + if ((def == null || !def.getLabel().equals(label)) && null != StudyManager.getInstance().getDatasetDefinitionByLabel(study, label)) + throw new IllegalArgumentException("A Dataset already exists with the label \"" + label +"\"."); + + // Additional key related exceptions + + if ("".equals(keyPropertyName)) + throw new IllegalArgumentException("Please select a field name for the additional key."); + + if (isManagedField && (keyPropertyName == null || keyPropertyName.isEmpty())) + throw new IllegalArgumentException("Additional Key Column name must be specified if field is managed."); + + if (useTimeKeyField && isManagedField) + throw new IllegalArgumentException("Additional key cannot be a managed field if KeyPropertyName is Time (from Date/Time)."); + + if (useTimeKeyField && !(keyPropertyName == null || keyPropertyName.equals(TIME_KEY_FIELD_KEY))) + throw new IllegalArgumentException("KeyPropertyName should not be provided when using additional key of Time (from Date/Time)."); + + if (isDemographicData && (isManagedField || keyPropertyName != null)) + throw new IllegalArgumentException("There cannot be an Additional Key Column if the dataset is Demographic Data."); + + if (!useTimeKeyField && null != keyPropertyName && null == domain.getFieldByName(keyPropertyName)) + throw new IllegalArgumentException("Additional Key Column name \"" + keyPropertyName +"\" must be the name of a column."); + + if (null != keyPropertyName && isManagedField) + { + String rangeURI = domain.getFieldByName(keyPropertyName).getRangeURI(); + if (!(rangeURI.endsWith("int") || rangeURI.endsWith("double") || rangeURI.endsWith("string"))) + throw new IllegalArgumentException("If Additional Key Column is managed, the column type must be numeric or text-based."); + } + + // Other exception(s) + + if (null != datasetId && (null == def || def.getDatasetId() != datasetId) && null != study.getDataset(datasetId)) + throw new IllegalArgumentException("A Dataset already exists with the datasetId \"" + datasetId +"\"."); + + if (!study.getShareVisitDefinitions() && null != datasetProperties.getDataSharing() && !datasetProperties.getDataSharing().equals("NONE")) + throw new IllegalArgumentException("Illegal value set for data sharing option."); + } + + private void checkCanUpdate(DatasetDefinition def, Container container, User user, DatasetDomainKindProperties datasetProperties, + GWTDomain original, GWTDomain update) + { + if (null == def) + throw new IllegalArgumentException("Dataset not found."); + + if (!def.canUpdateDefinition(user)) + throw new IllegalArgumentException("Shared dataset can not be edited in this folder."); + + if (datasetProperties.getLabel() == null || datasetProperties.getLabel().isEmpty()) + throw new IllegalArgumentException("Dataset label cannot be empty."); + + if (null == PropertyService.get().getDomain(container, update.getDomainURI())) + throw new IllegalArgumentException("Domain not found: " + update.getDomainURI() + "."); + + if (!def.getTypeURI().equals(original.getDomainURI()) || !def.getTypeURI().equals(update.getDomainURI())) + throw new IllegalArgumentException("Illegal Argument"); + + if (datasetProperties.isDemographicData() && !def.isDemographicData() && !StudyManager.getInstance().isDataUniquePerParticipant(def)) + { + String noun = StudyService.get().getSubjectNounSingular(container); + throw new IllegalArgumentException("This dataset currently contains more than one row of data per " + noun + + ". Demographic data includes one row of data per " + noun + "."); + } + } + + private @NotNull ValidationException updateDomainDescriptor(GWTDomain original, GWTDomain update, + @Nullable DatasetDefinition oldDef, @Nullable DatasetDomainKindProperties datasetPropertiesUpdate, Container container, User user, String userComment) + { + StringBuilder changeDetails = new StringBuilder(); + boolean hasNameChange = false; + Map oldProps = null; + Map newProps = datasetPropertiesUpdate != null ? datasetPropertiesUpdate.getAuditRecordMap() : null; + + if (oldDef != null) + { + oldProps = oldDef.getAuditRecordMap(); + if (newProps == null) + newProps = oldProps; // no update + + hasNameChange = !datasetPropertiesUpdate.getName().equals(oldDef.getName()); + if (hasNameChange) + changeDetails.append("The name of the dataset '" + oldDef.getName() + "' was changed to '" + datasetPropertiesUpdate.getName() + "'."); + } + + ValidationException exception = new ValidationException(); + exception.addErrors(DomainUtil.updateDomainDescriptor(original, update, container, user, hasNameChange, changeDetails.toString(), userComment, oldProps, newProps)); + return exception; + } + + private ValidationException updateDataset(DatasetDomainKindProperties datasetProperties, String domainURI, ValidationException exception, + StudyImpl study, Container container, User user, DatasetDefinition def) + { + try + { + // Check for usage of Time as Key Field + boolean useTimeKeyField = datasetProperties.isUseTimeKeyField(); + if (useTimeKeyField) + datasetProperties.setKeyPropertyName(null); + + // Default is no key management + KeyManagementType keyType = KeyManagementType.None; + String keyPropertyName = datasetProperties.getKeyPropertyName(); + if (datasetProperties.getKeyPropertyName() != null) + { + Domain domain = PropertyService.get().getDomain(container, domainURI); + for (DomainProperty dp : domain.getProperties()) + { + if (dp.getName().equalsIgnoreCase(datasetProperties.getKeyPropertyName())) + { + keyPropertyName = dp.getName(); + + // Be sure that the user really wants a managed key, not just that disabled select box still had a value + if (datasetProperties.isKeyPropertyManaged()) + keyType = KeyManagementType.getManagementTypeFromProp(dp.getPropertyDescriptor().getPropertyType()); + + break; + } + } + } + + DatasetDefinition updated = def.createMutable(); + + // Clear the category ID so that it gets regenerated based on the new string - see issue 19649 + updated.setCategoryId(null); + + BeanUtils.copyProperties(updated, datasetProperties); + updated.setKeyPropertyName(keyPropertyName); + updated.setKeyManagementType(keyType); + updated.setUseTimeKeyField(useTimeKeyField); + + List errors = new ArrayList<>(); + StudyManager.getInstance().updateDatasetDefinition(user, updated, errors); + StudyManager.datasetModified(updated, true); + + for (String errorMsg: errors) + exception.addGlobalError(errorMsg); + return exception; + } + catch (RuntimeSQLException e) + { + return exception.addGlobalError("Additional key column must have unique values."); + } + catch (Exception e) + { + throw new RuntimeException(e); + } + } + + @Override + public @NotNull ValidationException updateDomain(GWTDomain original, GWTDomain update, + @Nullable DatasetDomainKindProperties datasetProperties, Container container, User user, boolean includeWarnings, String userComment) + { + assert original.getDomainURI().equals(update.getDomainURI()); + StudyImpl study = StudyManager.getInstance().getStudy(container); + DatasetDefinition def = null; + boolean hasNameChange = false; + + if (datasetProperties != null) + { + def = study.getDataset(datasetProperties.getDatasetId()); + validateDatasetProperties(datasetProperties, container, user, update, def); + checkCanUpdate(def, container, user, datasetProperties, original, update); + String updatedName = StringUtils.trimToEmpty(datasetProperties.getName()); + datasetProperties.setName(updatedName); + hasNameChange = !def.getName().equals(updatedName); + } + + // Acquire lock before we actually start the transaction to avoid deadlocks when it's refreshed during the process + Lock[] locks = def == null ? new Lock[0] : new Lock[] { def.getDomainLoadingLock() }; + try (DbScope.Transaction transaction = StudySchema.getInstance().getScope().ensureTransaction(locks)) + { + ValidationException exception = updateDomainDescriptor(original, update, def, datasetProperties, container, user, userComment); + + QueryService.get().saveCalculatedFieldsMetadata("study", update.getQueryName(), hasNameChange ? datasetProperties.getName() : null, update.getCalculatedFields(), !original.getCalculatedFields().isEmpty(), user, container); + + if (!exception.hasErrors() && def != null) + exception = updateDataset(datasetProperties, original.getDomainURI(), exception, study, container, user, def); + + if (!exception.hasErrors()) + transaction.commit(); + + return exception; + } + catch (MetadataUnavailableException e) + { + return new ValidationException(e.getMessage()); + } + finally + { + if (def != null) + StudyManager.getInstance().uncache(def); + } + } + + @Override + public void deleteDomain(User user, Domain domain, @Nullable String auditUserComment) + { + DatasetDefinition def = StudyManager.getInstance().getDatasetDefinition(domain.getTypeURI()); + if (def == null) + throw new NotFoundException("Dataset not found: " + domain.getTypeURI()); + + StudyImpl study = StudyManager.getInstance().getStudy(domain.getContainer()); + if (study == null) + throw new IllegalArgumentException("A study does not exist for this folder"); + + try (DbScope.Transaction transaction = StudySchema.getInstance().getSchema().getScope().ensureTransaction()) + { + StudyManager.getInstance().deleteDataset(study, user, def, false, auditUserComment); + transaction.commit(); + } + } + + @Override + public boolean isDeleteAllDataOnFieldImport() + { + return true; + } + + + @Override + public TableInfo getTableInfo(User user, Container container, String name, @Nullable ContainerFilter cf) + { + StudyImpl study = StudyManager.getInstance().getStudy(container); + if (null == study) + return null; + StudyQuerySchema schema = StudyQuerySchema.createSchema(study, user); + DatasetDefinition dsd = schema.getDatasetDefinitionByName(name); + if (null == dsd) + return null; + + return schema.getTable(name, cf, true, false); + } + + @Override + public void afterLoadTable(SchemaTableInfo ti, Domain domain) + { + // Grab the "standard" properties and apply them to this dataset table + TableInfo template = DatasetDefinition.getTemplateTableInfo(); + + for (PropertyStorageSpec pss : domain.getDomainKind().getBaseProperties(domain)) + { + ColumnInfo c = ti.getColumn(pss.getName()); + ColumnInfo tCol = template.getColumn(pss.getName()); + // The column may be null if the dataset is being deleted in the background + if (null != tCol && c != null) + { + ((BaseColumnInfo)c).setExtraAttributesFrom(tCol); + + // When copying a column, the hidden bit is not propagated, so we need to do it manually + if (tCol.isHidden()) + ((BaseColumnInfo)c).setHidden(true); + } + } + } + + @Override + public boolean supportsPhiLevel() + { + return ComplianceService.get().isComplianceSupported(); + } + + // Query datasets don't have a domain kind, so check here if the dataset is a query dataset + @Override + public boolean isProvisioned(Container container, String name) + { + return super.isProvisioned(container, name) && !isQueryDataset(container, name); + } + + public static boolean isQueryDataset(Container container, String queryName) + { + StudyService ss = StudyService.get(); + if (ss == null) + return false; + + Dataset dt = ss.getDataset(container, ss.getDatasetIdByName(container, queryName)); + if (dt == null) + return false; + + return dt.isQueryDataset(); + } +} From 322bd919e5f78ed404a72f7f67281eb336d71a71 Mon Sep 17 00:00:00 2001 From: XingY Date: Mon, 12 Jan 2026 10:54:19 -0800 Subject: [PATCH 2/6] Multi value text choices --- api/schemas/queryCustomView.xsd | 6 ++++++ api/webapp/clientapi/ext3/FilterDialog.js | 2 ++ assay/package-lock.json | 16 ++++++++-------- assay/package.json | 2 +- core/package-lock.json | 16 ++++++++-------- core/package.json | 2 +- experiment/package-lock.json | 16 ++++++++-------- experiment/package.json | 2 +- pipeline/package-lock.json | 16 ++++++++-------- pipeline/package.json | 2 +- 10 files changed, 44 insertions(+), 36 deletions(-) diff --git a/api/schemas/queryCustomView.xsd b/api/schemas/queryCustomView.xsd index b0abe2a107d..61983586694 100644 --- a/api/schemas/queryCustomView.xsd +++ b/api/schemas/queryCustomView.xsd @@ -93,6 +93,12 @@ + + + + + + diff --git a/api/webapp/clientapi/ext3/FilterDialog.js b/api/webapp/clientapi/ext3/FilterDialog.js index 7e85ba94490..b4c25180a66 100644 --- a/api/webapp/clientapi/ext3/FilterDialog.js +++ b/api/webapp/clientapi/ext3/FilterDialog.js @@ -916,6 +916,8 @@ LABKEY.FilterDialog.View.Default = Ext.extend(LABKEY.FilterDialog.ViewPanel, { }, validateMultiValueInput : function(inputValues, multiValueSeparator, minOccurs, maxOccurs) { + if (!inputValues) + return true; // Used when "Equals One Of.." or "Between" is selected. Calls validateInputField on each value entered. const sep = inputValues.indexOf('\n') > 0 ? '\n' : multiValueSeparator; var values = inputValues.split(sep); diff --git a/assay/package-lock.json b/assay/package-lock.json index 4843a841ff3..155452c9304 100644 --- a/assay/package-lock.json +++ b/assay/package-lock.json @@ -8,7 +8,7 @@ "name": "assay", "version": "0.0.0", "dependencies": { - "@labkey/components": "7.12.0" + "@labkey/components": "7.13.0-fb-mvtc.4" }, "devDependencies": { "@labkey/build": "8.7.0", @@ -2482,9 +2482,9 @@ } }, "node_modules/@labkey/api": { - "version": "1.44.1", - "resolved": "https://labkey.jfrog.io/artifactory/api/npm/libs-client/@labkey/api/-/@labkey/api-1.44.1.tgz", - "integrity": "sha512-VUS4KLfwAsE45A3MnJUU3j97ei0ncQHv6OVVAN3kitID0xe8+mZ7B39zETVye3Dqgwa8TbYvsCp2t46QmBmwVQ==", + "version": "1.45.0-fb-mvtc.2", + "resolved": "https://labkey.jfrog.io/artifactory/api/npm/libs-client/@labkey/api/-/@labkey/api-1.45.0-fb-mvtc.2.tgz", + "integrity": "sha512-26xRLDWTZTOIGvoseGCeLVzMlwftHs6oxOlft6uhdmJpYBdm6+AyK2w+jU0Fns5SaEtaZAGofrBJFi8zCXttCg==", "license": "Apache-2.0" }, "node_modules/@labkey/build": { @@ -2525,13 +2525,13 @@ } }, "node_modules/@labkey/components": { - "version": "7.12.0", - "resolved": "https://labkey.jfrog.io/artifactory/api/npm/libs-client/@labkey/components/-/@labkey/components-7.12.0.tgz", - "integrity": "sha512-iwB+3m7JWcwJ7sPA8V+lPXeW6J0tSq/uueiRJ7EzzuaBubXgYwX4ISpZNVt4MzxtAybImzOGLU4ef4+2NFyVRw==", + "version": "7.13.0-fb-mvtc.4", + "resolved": "https://labkey.jfrog.io/artifactory/api/npm/libs-client/@labkey/components/-/@labkey/components-7.13.0-fb-mvtc.4.tgz", + "integrity": "sha512-eMcwnnL0LugsMWQpxh76H1RN5rW0lH3iCpk69oUOPRi/DqSGthA0xJsWbjYmrlK3yQQQiS1Pj+Efn35haKay+w==", "license": "SEE LICENSE IN LICENSE.txt", "dependencies": { "@hello-pangea/dnd": "18.0.1", - "@labkey/api": "1.44.1", + "@labkey/api": "1.45.0-fb-mvtc.2", "@testing-library/dom": "~10.4.1", "@testing-library/jest-dom": "~6.9.1", "@testing-library/react": "~16.3.0", diff --git a/assay/package.json b/assay/package.json index bba449b3803..a6da49980b9 100644 --- a/assay/package.json +++ b/assay/package.json @@ -12,7 +12,7 @@ "clean": "rimraf resources/web/assay/gen && rimraf resources/views/gen && rimraf resources/web/gen" }, "dependencies": { - "@labkey/components": "7.12.0" + "@labkey/components": "7.13.0-fb-mvtc.4" }, "devDependencies": { "@labkey/build": "8.7.0", diff --git a/core/package-lock.json b/core/package-lock.json index 5c3d14eb20d..40f9b507d54 100644 --- a/core/package-lock.json +++ b/core/package-lock.json @@ -8,7 +8,7 @@ "name": "labkey-core", "version": "0.0.0", "dependencies": { - "@labkey/components": "7.12.0", + "@labkey/components": "7.13.0-fb-mvtc.4", "@labkey/themes": "1.5.0" }, "devDependencies": { @@ -3504,9 +3504,9 @@ } }, "node_modules/@labkey/api": { - "version": "1.44.1", - "resolved": "https://labkey.jfrog.io/artifactory/api/npm/libs-client/@labkey/api/-/@labkey/api-1.44.1.tgz", - "integrity": "sha512-VUS4KLfwAsE45A3MnJUU3j97ei0ncQHv6OVVAN3kitID0xe8+mZ7B39zETVye3Dqgwa8TbYvsCp2t46QmBmwVQ==", + "version": "1.45.0-fb-mvtc.2", + "resolved": "https://labkey.jfrog.io/artifactory/api/npm/libs-client/@labkey/api/-/@labkey/api-1.45.0-fb-mvtc.2.tgz", + "integrity": "sha512-26xRLDWTZTOIGvoseGCeLVzMlwftHs6oxOlft6uhdmJpYBdm6+AyK2w+jU0Fns5SaEtaZAGofrBJFi8zCXttCg==", "license": "Apache-2.0" }, "node_modules/@labkey/build": { @@ -3547,13 +3547,13 @@ } }, "node_modules/@labkey/components": { - "version": "7.12.0", - "resolved": "https://labkey.jfrog.io/artifactory/api/npm/libs-client/@labkey/components/-/@labkey/components-7.12.0.tgz", - "integrity": "sha512-iwB+3m7JWcwJ7sPA8V+lPXeW6J0tSq/uueiRJ7EzzuaBubXgYwX4ISpZNVt4MzxtAybImzOGLU4ef4+2NFyVRw==", + "version": "7.13.0-fb-mvtc.4", + "resolved": "https://labkey.jfrog.io/artifactory/api/npm/libs-client/@labkey/components/-/@labkey/components-7.13.0-fb-mvtc.4.tgz", + "integrity": "sha512-eMcwnnL0LugsMWQpxh76H1RN5rW0lH3iCpk69oUOPRi/DqSGthA0xJsWbjYmrlK3yQQQiS1Pj+Efn35haKay+w==", "license": "SEE LICENSE IN LICENSE.txt", "dependencies": { "@hello-pangea/dnd": "18.0.1", - "@labkey/api": "1.44.1", + "@labkey/api": "1.45.0-fb-mvtc.2", "@testing-library/dom": "~10.4.1", "@testing-library/jest-dom": "~6.9.1", "@testing-library/react": "~16.3.0", diff --git a/core/package.json b/core/package.json index 49e27874a49..cc3e4bfcc32 100644 --- a/core/package.json +++ b/core/package.json @@ -53,7 +53,7 @@ } }, "dependencies": { - "@labkey/components": "7.12.0", + "@labkey/components": "7.13.0-fb-mvtc.4", "@labkey/themes": "1.5.0" }, "devDependencies": { diff --git a/experiment/package-lock.json b/experiment/package-lock.json index d09c9ad5605..c18b82920f4 100644 --- a/experiment/package-lock.json +++ b/experiment/package-lock.json @@ -8,7 +8,7 @@ "name": "experiment", "version": "0.0.0", "dependencies": { - "@labkey/components": "7.12.0" + "@labkey/components": "7.13.0-fb-mvtc.4" }, "devDependencies": { "@labkey/build": "8.7.0", @@ -3271,9 +3271,9 @@ } }, "node_modules/@labkey/api": { - "version": "1.44.1", - "resolved": "https://labkey.jfrog.io/artifactory/api/npm/libs-client/@labkey/api/-/@labkey/api-1.44.1.tgz", - "integrity": "sha512-VUS4KLfwAsE45A3MnJUU3j97ei0ncQHv6OVVAN3kitID0xe8+mZ7B39zETVye3Dqgwa8TbYvsCp2t46QmBmwVQ==", + "version": "1.45.0-fb-mvtc.2", + "resolved": "https://labkey.jfrog.io/artifactory/api/npm/libs-client/@labkey/api/-/@labkey/api-1.45.0-fb-mvtc.2.tgz", + "integrity": "sha512-26xRLDWTZTOIGvoseGCeLVzMlwftHs6oxOlft6uhdmJpYBdm6+AyK2w+jU0Fns5SaEtaZAGofrBJFi8zCXttCg==", "license": "Apache-2.0" }, "node_modules/@labkey/build": { @@ -3314,13 +3314,13 @@ } }, "node_modules/@labkey/components": { - "version": "7.12.0", - "resolved": "https://labkey.jfrog.io/artifactory/api/npm/libs-client/@labkey/components/-/@labkey/components-7.12.0.tgz", - "integrity": "sha512-iwB+3m7JWcwJ7sPA8V+lPXeW6J0tSq/uueiRJ7EzzuaBubXgYwX4ISpZNVt4MzxtAybImzOGLU4ef4+2NFyVRw==", + "version": "7.13.0-fb-mvtc.4", + "resolved": "https://labkey.jfrog.io/artifactory/api/npm/libs-client/@labkey/components/-/@labkey/components-7.13.0-fb-mvtc.4.tgz", + "integrity": "sha512-eMcwnnL0LugsMWQpxh76H1RN5rW0lH3iCpk69oUOPRi/DqSGthA0xJsWbjYmrlK3yQQQiS1Pj+Efn35haKay+w==", "license": "SEE LICENSE IN LICENSE.txt", "dependencies": { "@hello-pangea/dnd": "18.0.1", - "@labkey/api": "1.44.1", + "@labkey/api": "1.45.0-fb-mvtc.2", "@testing-library/dom": "~10.4.1", "@testing-library/jest-dom": "~6.9.1", "@testing-library/react": "~16.3.0", diff --git a/experiment/package.json b/experiment/package.json index d1aacc6542c..304af0c1255 100644 --- a/experiment/package.json +++ b/experiment/package.json @@ -13,7 +13,7 @@ "test-integration": "cross-env NODE_ENV=test jest --ci --runInBand -c test/js/jest.config.integration.js" }, "dependencies": { - "@labkey/components": "7.12.0" + "@labkey/components": "7.13.0-fb-mvtc.4" }, "devDependencies": { "@labkey/build": "8.7.0", diff --git a/pipeline/package-lock.json b/pipeline/package-lock.json index 7ba89586f45..921467d259a 100644 --- a/pipeline/package-lock.json +++ b/pipeline/package-lock.json @@ -8,7 +8,7 @@ "name": "pipeline", "version": "0.0.0", "dependencies": { - "@labkey/components": "7.12.0" + "@labkey/components": "7.13.0-fb-mvtc.4" }, "devDependencies": { "@labkey/build": "8.7.0", @@ -2716,9 +2716,9 @@ } }, "node_modules/@labkey/api": { - "version": "1.44.1", - "resolved": "https://labkey.jfrog.io/artifactory/api/npm/libs-client/@labkey/api/-/@labkey/api-1.44.1.tgz", - "integrity": "sha512-VUS4KLfwAsE45A3MnJUU3j97ei0ncQHv6OVVAN3kitID0xe8+mZ7B39zETVye3Dqgwa8TbYvsCp2t46QmBmwVQ==", + "version": "1.45.0-fb-mvtc.2", + "resolved": "https://labkey.jfrog.io/artifactory/api/npm/libs-client/@labkey/api/-/@labkey/api-1.45.0-fb-mvtc.2.tgz", + "integrity": "sha512-26xRLDWTZTOIGvoseGCeLVzMlwftHs6oxOlft6uhdmJpYBdm6+AyK2w+jU0Fns5SaEtaZAGofrBJFi8zCXttCg==", "license": "Apache-2.0" }, "node_modules/@labkey/build": { @@ -2759,13 +2759,13 @@ } }, "node_modules/@labkey/components": { - "version": "7.12.0", - "resolved": "https://labkey.jfrog.io/artifactory/api/npm/libs-client/@labkey/components/-/@labkey/components-7.12.0.tgz", - "integrity": "sha512-iwB+3m7JWcwJ7sPA8V+lPXeW6J0tSq/uueiRJ7EzzuaBubXgYwX4ISpZNVt4MzxtAybImzOGLU4ef4+2NFyVRw==", + "version": "7.13.0-fb-mvtc.4", + "resolved": "https://labkey.jfrog.io/artifactory/api/npm/libs-client/@labkey/components/-/@labkey/components-7.13.0-fb-mvtc.4.tgz", + "integrity": "sha512-eMcwnnL0LugsMWQpxh76H1RN5rW0lH3iCpk69oUOPRi/DqSGthA0xJsWbjYmrlK3yQQQiS1Pj+Efn35haKay+w==", "license": "SEE LICENSE IN LICENSE.txt", "dependencies": { "@hello-pangea/dnd": "18.0.1", - "@labkey/api": "1.44.1", + "@labkey/api": "1.45.0-fb-mvtc.2", "@testing-library/dom": "~10.4.1", "@testing-library/jest-dom": "~6.9.1", "@testing-library/react": "~16.3.0", diff --git a/pipeline/package.json b/pipeline/package.json index f4f0a663aba..8e4b629b05f 100644 --- a/pipeline/package.json +++ b/pipeline/package.json @@ -14,7 +14,7 @@ "build-prod": "npm run clean && cross-env NODE_ENV=production PROD_SOURCE_MAP=source-map webpack --config node_modules/@labkey/build/webpack/prod.config.js --color --progress --profile" }, "dependencies": { - "@labkey/components": "7.12.0" + "@labkey/components": "7.13.0-fb-mvtc.4" }, "devDependencies": { "@labkey/build": "8.7.0", From 672a750f74a9d4759b483a72e85aa8cc974803ba Mon Sep 17 00:00:00 2001 From: XingY Date: Mon, 12 Jan 2026 13:07:19 -0800 Subject: [PATCH 3/6] fix empty array --- .../labkey/api/data/ConnectionWrapper.java | 2 +- .../api/data/SimpleConnectionWrapper.java | 2 +- .../labkey/api/data/dialect/SqlDialect.java | 49 +++++++++++++++++++ 3 files changed, 51 insertions(+), 2 deletions(-) diff --git a/api/src/org/labkey/api/data/ConnectionWrapper.java b/api/src/org/labkey/api/data/ConnectionWrapper.java index bf81461c15f..ee11f5ca5c2 100644 --- a/api/src/org/labkey/api/data/ConnectionWrapper.java +++ b/api/src/org/labkey/api/data/ConnectionWrapper.java @@ -975,7 +975,7 @@ public Array createArrayOf(String unused, Object[] array) throws SQLException try { SqlDialect dialect = _scope.getSqlDialect(); - String typeName = dialect.getJDBCArrayType(array[0]); + String typeName = dialect.getJDBCArrayType(array); return _connection.createArrayOf(typeName, array); } catch (SQLException e) diff --git a/api/src/org/labkey/api/data/SimpleConnectionWrapper.java b/api/src/org/labkey/api/data/SimpleConnectionWrapper.java index 8d8a4402d84..68004fdd162 100644 --- a/api/src/org/labkey/api/data/SimpleConnectionWrapper.java +++ b/api/src/org/labkey/api/data/SimpleConnectionWrapper.java @@ -40,7 +40,7 @@ public SimpleConnectionWrapper(Connection connection, DbScope scope) public Array createArrayOf(String unused, Object[] array) throws SQLException { SqlDialect dialect = _scope.getSqlDialect(); - String typeName = dialect.getJDBCArrayType(array[0]); + String typeName = dialect.getJDBCArrayType(array); return _connection.createArrayOf(typeName, array); } diff --git a/api/src/org/labkey/api/data/dialect/SqlDialect.java b/api/src/org/labkey/api/data/dialect/SqlDialect.java index 390adcc71a9..c68b4268bc4 100644 --- a/api/src/org/labkey/api/data/dialect/SqlDialect.java +++ b/api/src/org/labkey/api/data/dialect/SqlDialect.java @@ -1300,6 +1300,55 @@ public String getJDBCArrayType(Object object) return StringUtils.lowerCase(getSqlTypeNameFromObject(object)); } + public String getJDBCArrayType(Object[] array) + { + String typeName; + if (array.length == 0) + { + // Handle empty arrays by inferring the SQL element type from the Java component type. + // Primary target is String[0], but handle a reasonable set of common types defensively. + Class componentType = array.getClass().getComponentType(); + if (String.class.equals(componentType)) + { + // Use dialect mapping for a String instance + typeName = getJDBCArrayType(""); + } + else if (Integer.class.equals(componentType)) + { + typeName = getJDBCArrayType(Integer.valueOf(0)); + } + else if (Long.class.equals(componentType)) + { + typeName = getJDBCArrayType(Long.valueOf(0L)); + } + else if (Double.class.equals(componentType)) + { + typeName = getJDBCArrayType(Double.valueOf(0.0d)); + } + else if (Float.class.equals(componentType)) + { + typeName = getJDBCArrayType(Float.valueOf(0.0f)); + } + else if (Boolean.class.equals(componentType)) + { + typeName = getJDBCArrayType(Boolean.FALSE); + } + else + { + // Fallback to VARCHAR which is the safest for most text use-cases + typeName = getSqlTypeName(JdbcType.VARCHAR); + if (typeName != null) + typeName = typeName.toLowerCase(); + } + } + else + { + typeName = getJDBCArrayType(array[0]); + } + + return typeName; + } + public Collection getScriptWarnings(String name, String sql) { return Collections.emptyList(); From 11f042c6838651426056b48076b895f3316abbc6 Mon Sep 17 00:00:00 2001 From: XingY Date: Mon, 12 Jan 2026 13:16:33 -0800 Subject: [PATCH 4/6] crlf --- .../api/gwt/client/model/GWTDomain.java | 674 ++--- api/src/org/labkey/api/exp/PropertyType.java | 2450 ++++++++--------- .../labkey/api/exp/property/DomainKind.java | 878 +++--- api/src/org/labkey/api/settings/AppProps.java | 536 ++-- .../labkey/experiment/ExperimentModule.java | 2262 +++++++-------- .../org/labkey/list/model/ListDomainKind.java | 1500 +++++----- .../labkey/study/model/DatasetDomainKind.java | 1744 ++++++------ 7 files changed, 5022 insertions(+), 5022 deletions(-) diff --git a/api/gwtsrc/org/labkey/api/gwt/client/model/GWTDomain.java b/api/gwtsrc/org/labkey/api/gwt/client/model/GWTDomain.java index 2550585c35e..fc2e69896d7 100644 --- a/api/gwtsrc/org/labkey/api/gwt/client/model/GWTDomain.java +++ b/api/gwtsrc/org/labkey/api/gwt/client/model/GWTDomain.java @@ -1,337 +1,337 @@ -/* - * Copyright (c) 2018-2019 LabKey Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.labkey.api.gwt.client.model; - -import com.fasterxml.jackson.annotation.JsonIgnore; -import com.google.gwt.user.client.rpc.IsSerializable; -import lombok.Getter; -import lombok.Setter; -import org.labkey.api.gwt.client.DefaultValueType; -import org.labkey.api.gwt.client.util.PropertyUtil; - -import java.util.ArrayList; -import java.util.Collections; -import java.util.HashSet; -import java.util.List; -import java.util.Set; - -public class GWTDomain implements IsSerializable -{ - private String _ts; - @Getter @Setter private int domainId; - @Getter @Setter private String name; - @Getter @Setter private String domainURI; - @Getter @Setter private String domainKindName; - @Getter @Setter private String description; - @Getter @Setter private String container; - @Getter @Setter private boolean allowFileLinkProperties; - @Getter @Setter private boolean allowAttachmentProperties; - @Getter @Setter private boolean allowFlagProperties; - @Getter @Setter private boolean allowTextChoiceProperties; - @Getter @Setter private boolean allowMultiChoiceProperties; - @Getter @Setter private boolean allowSampleSubjectProperties; - @Getter @Setter private boolean allowTimepointProperties; - @Getter @Setter private boolean allowUniqueConstraintProperties; - @Getter @Setter private boolean allowCalculatedFields; - @Getter @Setter private boolean showDefaultValueSettings; - private DefaultValueType defaultDefaultValueType = null; - private DefaultValueType[] defaultValueOptions = new DefaultValueType[0]; - private List fields = new ArrayList<>(); - private List standardFields = null; - private List calculatedFields = null; - @Getter @Setter private List indices = new ArrayList<>(); - private String defaultValuesURL = null; - private Set mandatoryPropertyDescriptorNames = new HashSet<>(); - private Set reservedFieldNames = new HashSet<>(); - private Set reservedFieldNamePrefixes = new HashSet<>(); - private Set phiNotAllowedFieldNames = new HashSet<>(); - private Set excludeFromExportFieldNames = new HashSet<>(); - @Getter @Setter private boolean provisioned = false; - @Getter @Setter private List disabledSystemFields; - - // schema,query,template are not part of the domain, but it's handy to pass - // these values to the PropertiedEditor along with the GWTDomain. - // NOTE queryName is not necessarily == name - @Getter @Setter private String schemaName = null; - @Getter @Setter private String queryName = null; - @Getter @Setter private String templateDescription = null; // null if no template - @Getter @Setter private String instructions = null; - @Getter @Setter private boolean supportsPhiLevel = false; - - public GWTDomain() - { - } - - // deep clone constructor - public GWTDomain(GWTDomain src) - { - _ts = src._ts; - this.domainId = src.domainId; - this.name = src.name; - this.domainURI = src.domainURI; - this.domainKindName = src.domainKindName; - this.description = src.description; - this.disabledSystemFields = src.disabledSystemFields; - this.container = src.container; - this.allowFileLinkProperties = src.allowFileLinkProperties; - this.allowAttachmentProperties = src.allowAttachmentProperties; - this.allowFlagProperties = src.allowFlagProperties; - this.allowTextChoiceProperties = src.allowTextChoiceProperties; - this.allowMultiChoiceProperties = src.allowMultiChoiceProperties; - this.allowSampleSubjectProperties = src.allowSampleSubjectProperties; - this.allowTimepointProperties = src.allowTimepointProperties; - this.allowUniqueConstraintProperties = src.allowUniqueConstraintProperties; - this.allowCalculatedFields = src.allowCalculatedFields; - this.showDefaultValueSettings = src.showDefaultValueSettings; - this.defaultDefaultValueType = src.defaultDefaultValueType; - this.defaultValueOptions = src.defaultValueOptions; - this.defaultValuesURL = src.defaultValuesURL; - this.provisioned = src.provisioned; - this.supportsPhiLevel = src.supportsPhiLevel; - - if (src.indices != null) - { - for (int i = 0; i < src.indices.size(); i++) - this.indices.add(src.indices.get(i).copy()); - } - - // include all fields here (standard and calculated) in the copy - if (src.getFields(true) == null) - return; - for (int i=0 ; i getFields(boolean includeCalculated) - { - if (includeCalculated) - return fields; - else - return getFields(); - } - - public List getFields() - { - if (standardFields == null) - standardFields = fields.stream().filter(f -> f.getValueExpression() == null).toList(); - return standardFields; - } - - public List getCalculatedFields() - { - if (calculatedFields == null) - calculatedFields = fields.stream().filter(f -> f.getValueExpression() != null).toList(); - return calculatedFields; - } - - public void setFields(List list) - { - fields = list; - - // reset the cached lists of fields so they will be recalculated on next call to getters - standardFields = null; - calculatedFields = null; - } - - public FieldType getFieldByName(String name) - { - for (FieldType field : getFields(true)) - { - if (field.getName() != null && field.getName().equalsIgnoreCase(name)) - return field; - } - return null; - } - - /** - * @return Indicates that the property can't be removed from the domain. The property may or may not be nullable. - */ - public boolean isMandatoryField(FieldType field) - { - if (mandatoryPropertyDescriptorNames == null || field.getName() == null) - { - return false; - } - return mandatoryPropertyDescriptorNames.contains(field.getName().toLowerCase()); - } - - public boolean isEditable(FieldType field) - { - return true; - } - - /** - * @return Indicates that the property is not allowed to be set as PHI - */ - public boolean allowsPhi(FieldType field) - { - return !(getPhiNotAllowedFieldNames() != null && field.getName() != null && getPhiNotAllowedFieldNames().contains(field.getName().toLowerCase())); - } - - /** - * @param mandatoryFieldNames names of property descriptors that must be present in this domain. Does not indicate that they must be non-nullable. - */ - public void setMandatoryFieldNames(Set mandatoryFieldNames) - { - this.mandatoryPropertyDescriptorNames = new HashSet<>(); - for (String mandatoryPropertyDescriptor : mandatoryFieldNames) - { - this.mandatoryPropertyDescriptorNames.add(mandatoryPropertyDescriptor.toLowerCase()); - } - } - - /** - * Get the list of property names that can't be removed from the domain. The set of mandatory fields is not modifiable in the designer. - */ - public Set getMandatoryFieldNames() - { - if (this.mandatoryPropertyDescriptorNames == null) - return Collections.emptySet(); - return Collections.unmodifiableSet(this.mandatoryPropertyDescriptorNames); - } - - public Set getReservedFieldNames() - { - return reservedFieldNames; - } - - /** - * @param reservedFieldNames can't create new fields with these names - */ - public void setReservedFieldNames(Set reservedFieldNames) - { - this.reservedFieldNames = new HashSet<>(); - for (String s : reservedFieldNames) - { - this.reservedFieldNames.add(s.toLowerCase()); - } - } - - public Set getReservedFieldNamePrefixes() - { - return this.reservedFieldNamePrefixes; - } - - public void setReservedFieldNamePrefixes(Set prefixes) - { - this.reservedFieldNamePrefixes = new HashSet<>(prefixes); - } - /** - * - * @param excludeFromExportFieldNames These fields will be suppressed from the export field list. Primary use case is to not export List key fields. - */ - public void setExcludeFromExportFieldNames(Set excludeFromExportFieldNames) - { - this.excludeFromExportFieldNames = new HashSet<>(); - for (String excludeFromExportFieldName : excludeFromExportFieldNames) - { - this.excludeFromExportFieldNames.add(excludeFromExportFieldName.toLowerCase()); - } - } - - public Set getExcludeFromExportFieldNames() - { - return excludeFromExportFieldNames; - } - - public boolean isExcludeFromExportField(FieldType field) - { - if (excludeFromExportFieldNames == null || field.getName() == null) - { - return false; - } - return excludeFromExportFieldNames.contains(field.getName().toLowerCase()); - } - - public Set getPhiNotAllowedFieldNames() - { - return phiNotAllowedFieldNames; - } - - public void setPhiNotAllowedFieldNames(Set phiNotAllowedFieldNames) - { - this.phiNotAllowedFieldNames = new HashSet<>(); - for (String fieldName : phiNotAllowedFieldNames) - { - this.phiNotAllowedFieldNames.add(fieldName.toLowerCase()); - } - } - - public DefaultValueType getDefaultDefaultValueType() - { - return defaultDefaultValueType; - } - - public DefaultValueType[] getDefaultValueOptions() - { - return defaultValueOptions; - } - - public void setDefaultValueOptions(DefaultValueType[] defaultOptions, DefaultValueType defaultDefault) - { - this.defaultDefaultValueType = defaultDefault; - this.defaultValueOptions = defaultOptions; - } - - public String getDefaultValuesURL() - { - if (defaultValuesURL == null) - return PropertyUtil.getRelativeURL("setDefaultValuesList", "list"); - return defaultValuesURL; - } - - public void setDefaultValuesURL(String defaultValuesURL) - { - this.defaultValuesURL = defaultValuesURL; - } -} +/* + * Copyright (c) 2018-2019 LabKey Corporation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.labkey.api.gwt.client.model; + +import com.fasterxml.jackson.annotation.JsonIgnore; +import com.google.gwt.user.client.rpc.IsSerializable; +import lombok.Getter; +import lombok.Setter; +import org.labkey.api.gwt.client.DefaultValueType; +import org.labkey.api.gwt.client.util.PropertyUtil; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashSet; +import java.util.List; +import java.util.Set; + +public class GWTDomain implements IsSerializable +{ + private String _ts; + @Getter @Setter private int domainId; + @Getter @Setter private String name; + @Getter @Setter private String domainURI; + @Getter @Setter private String domainKindName; + @Getter @Setter private String description; + @Getter @Setter private String container; + @Getter @Setter private boolean allowFileLinkProperties; + @Getter @Setter private boolean allowAttachmentProperties; + @Getter @Setter private boolean allowFlagProperties; + @Getter @Setter private boolean allowTextChoiceProperties; + @Getter @Setter private boolean allowMultiChoiceProperties; + @Getter @Setter private boolean allowSampleSubjectProperties; + @Getter @Setter private boolean allowTimepointProperties; + @Getter @Setter private boolean allowUniqueConstraintProperties; + @Getter @Setter private boolean allowCalculatedFields; + @Getter @Setter private boolean showDefaultValueSettings; + private DefaultValueType defaultDefaultValueType = null; + private DefaultValueType[] defaultValueOptions = new DefaultValueType[0]; + private List fields = new ArrayList<>(); + private List standardFields = null; + private List calculatedFields = null; + @Getter @Setter private List indices = new ArrayList<>(); + private String defaultValuesURL = null; + private Set mandatoryPropertyDescriptorNames = new HashSet<>(); + private Set reservedFieldNames = new HashSet<>(); + private Set reservedFieldNamePrefixes = new HashSet<>(); + private Set phiNotAllowedFieldNames = new HashSet<>(); + private Set excludeFromExportFieldNames = new HashSet<>(); + @Getter @Setter private boolean provisioned = false; + @Getter @Setter private List disabledSystemFields; + + // schema,query,template are not part of the domain, but it's handy to pass + // these values to the PropertiedEditor along with the GWTDomain. + // NOTE queryName is not necessarily == name + @Getter @Setter private String schemaName = null; + @Getter @Setter private String queryName = null; + @Getter @Setter private String templateDescription = null; // null if no template + @Getter @Setter private String instructions = null; + @Getter @Setter private boolean supportsPhiLevel = false; + + public GWTDomain() + { + } + + // deep clone constructor + public GWTDomain(GWTDomain src) + { + _ts = src._ts; + this.domainId = src.domainId; + this.name = src.name; + this.domainURI = src.domainURI; + this.domainKindName = src.domainKindName; + this.description = src.description; + this.disabledSystemFields = src.disabledSystemFields; + this.container = src.container; + this.allowFileLinkProperties = src.allowFileLinkProperties; + this.allowAttachmentProperties = src.allowAttachmentProperties; + this.allowFlagProperties = src.allowFlagProperties; + this.allowTextChoiceProperties = src.allowTextChoiceProperties; + this.allowMultiChoiceProperties = src.allowMultiChoiceProperties; + this.allowSampleSubjectProperties = src.allowSampleSubjectProperties; + this.allowTimepointProperties = src.allowTimepointProperties; + this.allowUniqueConstraintProperties = src.allowUniqueConstraintProperties; + this.allowCalculatedFields = src.allowCalculatedFields; + this.showDefaultValueSettings = src.showDefaultValueSettings; + this.defaultDefaultValueType = src.defaultDefaultValueType; + this.defaultValueOptions = src.defaultValueOptions; + this.defaultValuesURL = src.defaultValuesURL; + this.provisioned = src.provisioned; + this.supportsPhiLevel = src.supportsPhiLevel; + + if (src.indices != null) + { + for (int i = 0; i < src.indices.size(); i++) + this.indices.add(src.indices.get(i).copy()); + } + + // include all fields here (standard and calculated) in the copy + if (src.getFields(true) == null) + return; + for (int i=0 ; i getFields(boolean includeCalculated) + { + if (includeCalculated) + return fields; + else + return getFields(); + } + + public List getFields() + { + if (standardFields == null) + standardFields = fields.stream().filter(f -> f.getValueExpression() == null).toList(); + return standardFields; + } + + public List getCalculatedFields() + { + if (calculatedFields == null) + calculatedFields = fields.stream().filter(f -> f.getValueExpression() != null).toList(); + return calculatedFields; + } + + public void setFields(List list) + { + fields = list; + + // reset the cached lists of fields so they will be recalculated on next call to getters + standardFields = null; + calculatedFields = null; + } + + public FieldType getFieldByName(String name) + { + for (FieldType field : getFields(true)) + { + if (field.getName() != null && field.getName().equalsIgnoreCase(name)) + return field; + } + return null; + } + + /** + * @return Indicates that the property can't be removed from the domain. The property may or may not be nullable. + */ + public boolean isMandatoryField(FieldType field) + { + if (mandatoryPropertyDescriptorNames == null || field.getName() == null) + { + return false; + } + return mandatoryPropertyDescriptorNames.contains(field.getName().toLowerCase()); + } + + public boolean isEditable(FieldType field) + { + return true; + } + + /** + * @return Indicates that the property is not allowed to be set as PHI + */ + public boolean allowsPhi(FieldType field) + { + return !(getPhiNotAllowedFieldNames() != null && field.getName() != null && getPhiNotAllowedFieldNames().contains(field.getName().toLowerCase())); + } + + /** + * @param mandatoryFieldNames names of property descriptors that must be present in this domain. Does not indicate that they must be non-nullable. + */ + public void setMandatoryFieldNames(Set mandatoryFieldNames) + { + this.mandatoryPropertyDescriptorNames = new HashSet<>(); + for (String mandatoryPropertyDescriptor : mandatoryFieldNames) + { + this.mandatoryPropertyDescriptorNames.add(mandatoryPropertyDescriptor.toLowerCase()); + } + } + + /** + * Get the list of property names that can't be removed from the domain. The set of mandatory fields is not modifiable in the designer. + */ + public Set getMandatoryFieldNames() + { + if (this.mandatoryPropertyDescriptorNames == null) + return Collections.emptySet(); + return Collections.unmodifiableSet(this.mandatoryPropertyDescriptorNames); + } + + public Set getReservedFieldNames() + { + return reservedFieldNames; + } + + /** + * @param reservedFieldNames can't create new fields with these names + */ + public void setReservedFieldNames(Set reservedFieldNames) + { + this.reservedFieldNames = new HashSet<>(); + for (String s : reservedFieldNames) + { + this.reservedFieldNames.add(s.toLowerCase()); + } + } + + public Set getReservedFieldNamePrefixes() + { + return this.reservedFieldNamePrefixes; + } + + public void setReservedFieldNamePrefixes(Set prefixes) + { + this.reservedFieldNamePrefixes = new HashSet<>(prefixes); + } + /** + * + * @param excludeFromExportFieldNames These fields will be suppressed from the export field list. Primary use case is to not export List key fields. + */ + public void setExcludeFromExportFieldNames(Set excludeFromExportFieldNames) + { + this.excludeFromExportFieldNames = new HashSet<>(); + for (String excludeFromExportFieldName : excludeFromExportFieldNames) + { + this.excludeFromExportFieldNames.add(excludeFromExportFieldName.toLowerCase()); + } + } + + public Set getExcludeFromExportFieldNames() + { + return excludeFromExportFieldNames; + } + + public boolean isExcludeFromExportField(FieldType field) + { + if (excludeFromExportFieldNames == null || field.getName() == null) + { + return false; + } + return excludeFromExportFieldNames.contains(field.getName().toLowerCase()); + } + + public Set getPhiNotAllowedFieldNames() + { + return phiNotAllowedFieldNames; + } + + public void setPhiNotAllowedFieldNames(Set phiNotAllowedFieldNames) + { + this.phiNotAllowedFieldNames = new HashSet<>(); + for (String fieldName : phiNotAllowedFieldNames) + { + this.phiNotAllowedFieldNames.add(fieldName.toLowerCase()); + } + } + + public DefaultValueType getDefaultDefaultValueType() + { + return defaultDefaultValueType; + } + + public DefaultValueType[] getDefaultValueOptions() + { + return defaultValueOptions; + } + + public void setDefaultValueOptions(DefaultValueType[] defaultOptions, DefaultValueType defaultDefault) + { + this.defaultDefaultValueType = defaultDefault; + this.defaultValueOptions = defaultOptions; + } + + public String getDefaultValuesURL() + { + if (defaultValuesURL == null) + return PropertyUtil.getRelativeURL("setDefaultValuesList", "list"); + return defaultValuesURL; + } + + public void setDefaultValuesURL(String defaultValuesURL) + { + this.defaultValuesURL = defaultValuesURL; + } +} diff --git a/api/src/org/labkey/api/exp/PropertyType.java b/api/src/org/labkey/api/exp/PropertyType.java index d60ab77d3e2..50840fcda29 100644 --- a/api/src/org/labkey/api/exp/PropertyType.java +++ b/api/src/org/labkey/api/exp/PropertyType.java @@ -1,1225 +1,1225 @@ -/* - * Copyright (c) 2008-2019 LabKey Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.labkey.api.exp; - -import org.apache.commons.beanutils.ConversionException; -import org.apache.commons.beanutils.ConvertUtils; -import org.apache.poi.ss.usermodel.Cell; -import org.apache.poi.ss.usermodel.CellType; -import org.fhcrc.cpas.exp.xml.SimpleTypeNames; -import org.jetbrains.annotations.NotNull; -import org.jetbrains.annotations.Nullable; -import org.labkey.api.attachments.AttachmentFile; -import org.labkey.api.collections.CaseInsensitiveHashMap; -import org.labkey.api.data.JdbcType; -import org.labkey.api.data.MultiChoice; -import org.labkey.api.data.NameGenerator; -import org.labkey.api.exp.OntologyManager.PropertyRow; -import org.labkey.api.reader.ExcelFactory; -import org.labkey.api.util.DateUtil; -import org.labkey.vfs.FileLike; - -import java.io.File; -import java.math.BigDecimal; -import java.nio.ByteBuffer; -import java.sql.Array; -import java.sql.SQLException; -import java.sql.Time; -import java.text.DateFormat; -import java.text.ParseException; -import java.text.SimpleDateFormat; -import java.util.Date; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.Objects; -import java.util.TimeZone; - -import static org.labkey.api.util.IntegerUtils.asIntegerElseNull; -import static org.labkey.api.util.IntegerUtils.asLongElseNull; - -/** - * TODO: Add more types? Entity, Lsid, User, ... - */ -public enum PropertyType -{ - BOOLEAN("http://www.w3.org/2001/XMLSchema#boolean", "Boolean", 'f', JdbcType.BOOLEAN, 10, null, CellType.BOOLEAN, Boolean.class, Boolean.TYPE) - { - @Override - protected Object convertExcelValue(Cell cell) throws ConversionException - { - return cell.getBooleanCellValue(); - } - - @Override - public Object convert(Object value) throws ConversionException - { - boolean boolValue = false; - if (value instanceof Boolean) - boolValue = (Boolean)value; - else if (null != value && !"".equals(value)) - boolValue = (Boolean) ConvertUtils.convert(value.toString(), Boolean.class); - return boolValue; - } - - @Override - public SimpleTypeNames.Enum getXmlBeanType() - { - return SimpleTypeNames.BOOLEAN; - } - - @Override - protected void init(PropertyRow row, Object value) - { - Boolean b = (Boolean)value; - row.floatValue = b == Boolean.TRUE ? 1.0 : 0.0; - } - - @Override - protected void setValue(ObjectProperty property, Object value) - { - Boolean boolValue = null; - if (value instanceof Boolean) - boolValue = (Boolean)value; - else if (null != value) - boolValue = (Boolean) ConvertUtils.convert(value.toString(), Boolean.class); - property.floatValue = boolValue == null ? null : boolValue == Boolean.TRUE ? 1.0 : 0.0; - } - - @Override - protected Object getValue(ObjectProperty property) - { - return property.floatValue == null ? null : property.floatValue.intValue() != 0 ? Boolean.TRUE : Boolean.FALSE; - } - - @Override - public Object getPreviewValue(@Nullable String prefix) - { - return Boolean.TRUE; - } - }, - STRING("http://www.w3.org/2001/XMLSchema#string", "String", 's', JdbcType.VARCHAR, 4000, "text", CellType.STRING, String.class) - { - @Override - protected Object convertExcelValue(Cell cell) throws ConversionException - { - return cell.getStringCellValue(); - } - - @Override - public Object convert(Object value) throws ConversionException - { - if (value instanceof String) - return value; - else - return ConvertUtils.convert(value); - } - - @Override - public SimpleTypeNames.Enum getXmlBeanType() - { - return SimpleTypeNames.STRING; - } - - @Override - protected void init(PropertyRow row, Object value) - { - row.stringValue = (String)value; - } - - @Override - protected void setValue(ObjectProperty property, Object value) - { - property.stringValue = value == null ? null : value.toString(); - } - - @Override - protected Object getValue(ObjectProperty property) - { - return property.getStringValue(); - } - - @Override - public Object getPreviewValue(@Nullable String prefix) - { - return prefix + "Value"; - } - }, - // NOT an XMLSchema type uri??? - MULTI_LINE("http://www.w3.org/2001/XMLSchema#multiLine", "MultiLine", 's', JdbcType.VARCHAR, 4000, "textarea", CellType.STRING, String.class) - { - @Override - protected Object convertExcelValue(Cell cell) throws ConversionException - { - return cell.getStringCellValue(); - } - - @Override - public Object convert(Object value) throws ConversionException - { - if (value instanceof String) - return value; - else - return ConvertUtils.convert(value); - } - - @Override - public SimpleTypeNames.Enum getXmlBeanType() - { - return SimpleTypeNames.STRING; - } - - @Override - protected void init(PropertyRow row, Object value) - { - row.stringValue = (String)value; - } - - @Override - protected void setValue(ObjectProperty property, Object value) - { - property.stringValue = value == null ? null : value.toString(); - } - - @Override - protected Object getValue(ObjectProperty property) - { - return property.getStringValue(); - } - - @Override - public Object getPreviewValue(@Nullable String prefix) - { - return prefix + "Value"; - } - }, - MULTI_CHOICE("http://cpas.fhcrc.org/exp/xml#multiChoice", "MultiChoice", '?' /* unsupported in exp.PropertyValues */, JdbcType.ARRAY, 0, "textarea", CellType.STRING, List.class) - { - @Override - protected Object convertExcelValue(Cell cell) throws ConversionException - { - return ConvertUtils.convert(cell.getStringCellValue(), MultiChoice.Array.class); - } - - @Override - public Object convert(Object value) throws ConversionException - { - return MultiChoice.Converter.getInstance().convert(MultiChoice.Array.class, value); - } - - @Override - public SimpleTypeNames.Enum getXmlBeanType() - { - return SimpleTypeNames.STRING; - } - - @Override - protected void init(PropertyRow row, Object value) - { - throw new UnsupportedOperationException("TODO MultiChoice"); - } - - @Override - protected void setValue(ObjectProperty property, Object value) - { - if ((value instanceof java.sql.Array array)) - property.arrayValue = MultiChoice.Array.from(array); - else if (value != null) - property.arrayValue = MultiChoice.Array.from(new Object[]{value}); - } - - @Override - protected Object getValue(ObjectProperty property) - { - return property.arrayValue; - } - - @Override - public Object getPreviewValue(@Nullable String prefix) - { - return "Option 1, Option 2"; - } - }, - RESOURCE("http://www.w3.org/2000/01/rdf-schema#Resource", "PropertyURI", 's', JdbcType.VARCHAR, 4000, null, CellType.STRING, Identifiable.class) - { - @Override - protected Object convertExcelValue(Cell cell) throws ConversionException - { - return cell.getStringCellValue(); - } - - @Override - public Object convert(Object value) throws ConversionException - { - if (null == value) - return null; - if (value instanceof Identifiable) - return ((Identifiable) value).getLSID(); - else - return value.toString(); - } - - @Override - public SimpleTypeNames.Enum getXmlBeanType() - { - return SimpleTypeNames.STRING; - } - - @Override - protected void init(PropertyRow row, Object value) - { - row.stringValue = (String)value; - } - - @Override - protected void setValue(ObjectProperty property, Object value) - { - if (value instanceof Identifiable) - { - property.stringValue = ((Identifiable) value).getLSID(); - property.objectValue = (Identifiable) value; - } - else if (null != value) - property.stringValue = value.toString(); - } - - @Override - protected Object getValue(ObjectProperty property) - { - if (null != property.objectValue) - return property.objectValue; - else - return property.getStringValue(); - } - - @Override - public Object getPreviewValue(@Nullable String prefix) - { - return prefix + "Value"; - } - }, - INTEGER("http://www.w3.org/2001/XMLSchema#int", "Integer", 'f', JdbcType.INTEGER, 10, null, CellType.NUMERIC, Integer.class, Integer.TYPE, Long.class, Long.TYPE) - { - @Override - protected Object convertExcelValue(Cell cell) throws ConversionException - { - return (int)cell.getNumericCellValue(); - } - - @Override - public Object convert(Object value) throws ConversionException - { - if (null == value) - return null; - if (asIntegerElseNull(value) instanceof Integer i) - return i; - else - return ConvertUtils.convert(value.toString(), Integer.class); - } - - @Override - public SimpleTypeNames.Enum getXmlBeanType() - { - return SimpleTypeNames.INTEGER; - } - - @Override - protected void init(PropertyRow row, Object value) - { - Number n = (Number) value; - if (null != n) - row.floatValue = n.doubleValue(); - } - - @Override - protected void setValue(ObjectProperty property, Object value) - { - if (null == value) - property.floatValue = null; - else if (asIntegerElseNull(value) instanceof Integer i) - property.floatValue = i.doubleValue(); - else - property.floatValue = (Double) ConvertUtils.convert(value.toString(), Double.class); - } - - @Override - protected Object getValue(ObjectProperty property) - { - return property.floatValue == null ? null : property.floatValue.intValue(); - } - - @Override - public Object getPreviewValue(@Nullable String prefix) - { - return Integer.valueOf(3); - } - }, - BIGINT("http://www.w3.org/2001/XMLSchema#long", "Long", 'f', JdbcType.BIGINT, 10, null, CellType.NUMERIC, Long.class, Long.TYPE) - { - @Override - protected Object convertExcelValue(Cell cell) throws ConversionException - { - return (int)cell.getNumericCellValue(); - } - - @Override - public Object convert(Object value) throws ConversionException - { - if (null == value) - return null; - if (asLongElseNull(value) instanceof Long l) - return l; - else - return ConvertUtils.convert(value.toString(), Long.class); - } - - @Override - public SimpleTypeNames.Enum getXmlBeanType() - { - throw new UnsupportedOperationException(); - } - - @Override - protected void init(PropertyRow row, Object value) - { - Number n = (Number) value; - if (null != n) - row.floatValue = n.doubleValue(); - } - - @Override - protected void setValue(ObjectProperty property, Object value) - { - if (null == value) - property.floatValue = null; - else if (asLongElseNull(value) instanceof Long l) - property.floatValue = l.doubleValue(); - else - property.floatValue = (Double) ConvertUtils.convert(value.toString(), Double.class); - } - - @Override - protected Object getValue(ObjectProperty property) - { - return property.floatValue == null ? null : property.floatValue.longValue(); - } - - @Override - public Object getPreviewValue(@Nullable String prefix) - { - return Integer.valueOf(3); - } - }, - // NOT an XMLSchema type uri??? - BINARY("http://www.w3.org/2001/XMLSchema#binary", "Binary", 'f', JdbcType.BINARY, 10, null, CellType.NUMERIC, ByteBuffer.class) - { - @Override - protected Object convertExcelValue(Cell cell) throws ConversionException - { - return (int)cell.getNumericCellValue(); - } - - @Override - public Object convert(Object value) throws ConversionException - { - if (null == value) - return null; - if (value instanceof ByteBuffer) - return value; - else - return ConvertUtils.convert(value.toString(), ByteBuffer.class); - } - - @Override - public SimpleTypeNames.Enum getXmlBeanType() - { - throw new UnsupportedOperationException(); - } - - @Override - protected void init(PropertyRow row, Object value) - { - throw new UnsupportedOperationException(); - } - - @Override - protected void setValue(ObjectProperty property, Object value) - { - if (null != value) - property.floatValue = (Double) ConvertUtils.convert(value.toString(), Double.class); - } - - @Override - protected Object getValue(ObjectProperty property) - { - throw new UnsupportedOperationException(); - } - }, - /** Stored as a path to a file on the server's file system */ - FILE_LINK("http://cpas.fhcrc.org/exp/xml#fileLink", "FileLink", 's', JdbcType.VARCHAR, 400, "file", CellType.STRING, File.class) - { - @Override - protected Object convertExcelValue(Cell cell) throws ConversionException - { - return cell.getStringCellValue(); - } - - @Override - public Object convert(Object value) throws ConversionException - { - if (null == value) - return null; - if (value instanceof File) - return ((File) value).getPath(); - else - return String.valueOf(value); - } - - @Override - public SimpleTypeNames.Enum getXmlBeanType() - { - return SimpleTypeNames.FILE_LINK; - } - - @Override - protected void init(PropertyRow row, Object value) - { - row.stringValue = (String)value; - } - - @Override - protected void setValue(ObjectProperty property, Object value) - { - if (value instanceof File f) - property.stringValue = f.getPath(); - else if (value instanceof FileLike fl) - property.stringValue = fl.toNioPathForRead().toString(); - else - property.stringValue = value == null ? null : value.toString(); - } - - @Override - protected Object getValue(ObjectProperty property) - { - String value = property.getStringValue(); - return value == null ? null : new File(value); - } - - @Override - public Object getPreviewValue(@Nullable String prefix) - { - return prefix + "Value"; - } - }, - /** Stored in the database as a BLOB using AttachmentService */ - ATTACHMENT("http://www.labkey.org/exp/xml#attachment", "Attachment", 's', JdbcType.VARCHAR, 100, "file", CellType.STRING, File.class) - { - @Override - protected Object convertExcelValue(Cell cell) throws ConversionException - { - return cell.getStringCellValue(); - } - - @Override - public Object convert(Object value) throws ConversionException - { - if (null == value) - return null; - if (value instanceof File) - return ((File) value).getPath(); - else - return String.valueOf(value); - } - - @Override - public SimpleTypeNames.Enum getXmlBeanType() - { - throw new UnsupportedOperationException(); - } - - @Override - protected void init(PropertyRow row, Object value) - { - row.stringValue = (String)value; - } - - @Override - protected void setValue(ObjectProperty property, Object value) - { - if (value instanceof AttachmentFile) - { - property.stringValue = ((AttachmentFile)value).getFilename(); - } - else - property.stringValue = value == null ? null : value.toString(); - } - - @Override - protected Object getValue(ObjectProperty property) - { - return property.getStringValue(); - } - - @Override - public Object getPreviewValue(@Nullable String prefix) - { - return prefix + "Value"; - } - }, - DATE_TIME("http://www.w3.org/2001/XMLSchema#dateTime", "DateTime", 'd', JdbcType.TIMESTAMP, 100, null, CellType.NUMERIC, Date.class) - { - @Override - protected Object convertExcelValue(Cell cell) throws ConversionException - { - Date date = cell.getDateCellValue(); - if (date != null) - { - DateFormat format = new SimpleDateFormat("MM/dd/yyyy GG HH:mm:ss.SSS"); - format.setTimeZone(TimeZone.getDefault()); - String s = format.format(date); - try - { - date = format.parse(s); - } - catch (ParseException e) - { - throw new ConversionException(e); - } -// int offset = TimeZone.getDefault().getOffset(date.getTime()); -// date.setTime(date.getTime() - offset); - } - return date; - } - - @Override - public Object convert(Object value) throws ConversionException - { - if (null == value) - return null; - if (value instanceof Date) - return value; - else - { - String strVal = value.toString(); - if (DateUtil.isSignedDuration(strVal)) - strVal = JdbcType.TIMESTAMP.convert(value).toString(); - return ConvertUtils.convert(strVal, Date.class); - } - } - - @Override - public SimpleTypeNames.Enum getXmlBeanType() - { - return SimpleTypeNames.DATE_TIME; - } - - @Override - protected void init(PropertyRow row, Object value) - { - row.dateTimeValue = new java.sql.Time(((java.util.Date)value).getTime()); - } - - @Override - protected void setValue(ObjectProperty property, Object value) - { - if (value instanceof Date) - property.dateTimeValue = (Date) value; - else if (null != value) - property.dateTimeValue = (Date) ConvertUtils.convert(value.toString(), Date.class); - } - - @Override - protected Object getValue(ObjectProperty property) - { - return property.dateTimeValue; - } - - - @Override - public Object getPreviewValue(@Nullable String prefix) - { - return NameGenerator.PREVIEW_DATETIME_VALUE; - } - }, - DATE("http://www.w3.org/2001/XMLSchema#date", "Date", 'd', JdbcType.DATE, 100, null, CellType.NUMERIC, Date.class) - { - @Override - protected Object convertExcelValue(Cell cell) throws ConversionException - { - return DateUtil.getDateOnly((Date)DATE_TIME.convertExcelValue(cell)); - } - - @Override - public Object convert(Object value) throws ConversionException - { - return DateUtil.getDateOnly((Date)DATE_TIME.convert(value)); - } - - @Override - public SimpleTypeNames.Enum getXmlBeanType() - { - return SimpleTypeNames.DATE_TIME; - } - - @Override - protected void init(PropertyRow row, Object value) - { - row.dateTimeValue = new java.sql.Date(((java.util.Date)value).getTime()); - } - - @Override - protected void setValue(ObjectProperty property, Object value) - { - if (value instanceof Date) - property.dateTimeValue = (Date) value; - else if (null != value) - property.dateTimeValue = (Date) ConvertUtils.convert(value.toString(), Date.class); - } - - @Override - protected Object getValue(ObjectProperty property) - { - return property.dateTimeValue; - } - - @Override - public Object getPreviewValue(@Nullable String prefix) - { - return NameGenerator.PREVIEW_DATE_VALUE; - } - }, - TIME("http://www.w3.org/2001/XMLSchema#time", "Time", 'd', JdbcType.TIME, 100, null, CellType.NUMERIC, java.sql.Time.class) - { - @Override - protected Object convertExcelValue(Cell cell) throws ConversionException - { - return DateUtil.getTimeOnly((Date)DATE_TIME.convertExcelValue(cell)); - } - - @Override - public Object convert(Object value) throws ConversionException - { - if (null == value) - return null; - - if (value instanceof Time) - return value; - - if (value instanceof Date) - return DateUtil.getTimeOnly((Date) value); - - try - { - return ConvertUtils.convert(value, Time.class); - } - catch (Exception ignore) - { - } - - return DateUtil.getTimeOnly((Date)DATE_TIME.convert(value)); - } - - @Override - public SimpleTypeNames.Enum getXmlBeanType() - { - return SimpleTypeNames.DATE_TIME; - } - - @Override - protected void init(PropertyRow row, Object value) - { - row.dateTimeValue = new java.sql.Time(((java.util.Date)value).getTime()); - } - - @Override - protected void setValue(ObjectProperty property, Object value) - { - if (value instanceof Time) - property.dateTimeValue = (Time) value; - else if (null != value) - property.dateTimeValue = (Time) ConvertUtils.convert(value.toString(), Time.class); - } - - @Override - protected Object getValue(ObjectProperty property) - { - return property.dateTimeValue; - } - - @Override - public Object getPreviewValue(@Nullable String prefix) - { - return NameGenerator.PREVIEW_TIME_VALUE; - } - }, - DOUBLE("http://www.w3.org/2001/XMLSchema#double", "Double", 'f', JdbcType.DOUBLE, 20, null, CellType.NUMERIC, Double.class, Double.TYPE, Float.class, Float.TYPE) - { - @Override - protected Object convertExcelValue(Cell cell) throws ConversionException - { - return cell.getNumericCellValue(); - } - - @Override - public Object convert(Object value) throws ConversionException - { - if (null == value) - return null; - if (value instanceof Double) - return value; - else - return ConvertUtils.convert(String.valueOf(value), Double.class); - } - - @Override - public SimpleTypeNames.Enum getXmlBeanType() - { - return SimpleTypeNames.DOUBLE; - } - - @Override - protected void init(PropertyRow row, Object value) - { - Number n = (Number) value; - if (null != n) - row.floatValue = n.doubleValue(); - } - - @Override - protected void setValue(ObjectProperty property, Object value) - { - if (value instanceof Double) - property.floatValue = (Double) value; - else if (null != value) - property.floatValue = (Double) ConvertUtils.convert(value.toString(), Double.class); - } - - @Override - protected Object getValue(ObjectProperty property) - { - return property.floatValue; - } - - @Override - public Object getPreviewValue(@Nullable String prefix) - { - return 12.34; - } - }, - FLOAT("http://www.w3.org/2001/XMLSchema#float", "Float", 'f', JdbcType.REAL, 20, null, CellType.NUMERIC, Float.class, Float.TYPE) - { - @Override - protected Object convertExcelValue(Cell cell) throws ConversionException - { - return cell.getNumericCellValue(); - } - - @Override - public Object convert(Object value) throws ConversionException - { - if (null == value) - return null; - if (value instanceof Float) - return value; - else - return ConvertUtils.convert(String.valueOf(value), Float.class); - } - - @Override - public SimpleTypeNames.Enum getXmlBeanType() - { - throw new UnsupportedOperationException(); - } - - @Override - protected void init(PropertyRow row, Object value) - { - Number n = (Number) value; - if (null != n) - row.floatValue = n.doubleValue(); - } - - @Override - protected void setValue(ObjectProperty property, Object value) - { - if (value instanceof Double) - property.floatValue = (Double) value; - else if (null != value) - property.floatValue = (Double) ConvertUtils.convert(value.toString(), Double.class); - } - - @Override - protected Object getValue(ObjectProperty property) - { - return property.floatValue; - } - - @Override - public Object getPreviewValue(@Nullable String prefix) - { - return 12.34; - } - }, - DECIMAL("http://www.w3.org/2001/XMLSchema#decimal", "Decimal", 'f', JdbcType.DECIMAL, 20, null, CellType.NUMERIC, BigDecimal.class) - { - @Override - protected Object convertExcelValue(Cell cell) throws ConversionException - { - return cell.getNumericCellValue(); - } - - @Override - public Object convert(Object value) throws ConversionException - { - if (null == value) - return null; - if (value instanceof BigDecimal) - return value; - else - return ConvertUtils.convert(String.valueOf(value), BigDecimal.class); - } - - @Override - public SimpleTypeNames.Enum getXmlBeanType() - { - throw new UnsupportedOperationException(); - } - - @Override - protected void init(PropertyRow row, Object value) - { - Number n = (Number) value; - if (null != n) - row.floatValue = n.doubleValue(); - } - - @Override - protected void setValue(ObjectProperty property, Object value) - { - if (null != value) - property.floatValue = (Double) ConvertUtils.convert(value.toString(), Double.class); - } - - @Override - protected Object getValue(ObjectProperty property) - { - return property.floatValue; - } - - @Override - public Object getPreviewValue(@Nullable String prefix) - { - return 12.34; - } - }, - XML_TEXT("http://cpas.fhcrc.org/exp/xml#text-xml", "XmlText", 's', JdbcType.LONGVARCHAR, 4000, null, CellType.STRING, null) - { - @Override - protected Object convertExcelValue(Cell cell) throws ConversionException - { - return cell.getStringCellValue(); - } - - @Override - public Object convert(Object value) throws ConversionException - { - if (value instanceof String) - return value; - else - return ConvertUtils.convert(value); - } - - @Override - public SimpleTypeNames.Enum getXmlBeanType() - { - throw new UnsupportedOperationException(); - } - - @Override - protected void init(PropertyRow row, Object value) - { - throw new UnsupportedOperationException(); - } - - @Override - protected void setValue(ObjectProperty property, Object value) - { - throw new UnsupportedOperationException(); - } - - @Override - protected Object getValue(ObjectProperty property) - { - return property.getStringValue(); - } - - @Override - public Object getPreviewValue(@Nullable String prefix) - { - return prefix + "Value"; - } - }; - - private final String typeURI; - private final String xarName; - private final char storageType; - private final CellType excelCellType; - private final @NotNull JdbcType jdbcType; - private final int scale; - private final String inputType; - private final Class javaType; - private final Class[] additionalTypes; - - private static Map uriToProperty; - private static Map xarToProperty = null; - - PropertyType(String typeURI, - String xarName, - char storageType, - @NotNull JdbcType jdbcType, - int scale, - String inputType, - CellType excelCellType, - Class javaType, - Class... additionalTypes) - { - this.typeURI = typeURI; - this.xarName = xarName; - this.storageType = storageType; - this.jdbcType = jdbcType; - this.scale = scale; - this.inputType = inputType; - this.javaType = javaType; - this.excelCellType = excelCellType; - this.additionalTypes = additionalTypes; - } - - public String getTypeUri() - { - return typeURI; - } - - public String getXmlName() - { - return xarName; - } - - public char getStorageType() - { - return storageType; - } - - @NotNull - public JdbcType getJdbcType() - { - return jdbcType; - } - - public int getScale() - { - return scale; - } - - @Nullable - public String getInputType() - { - return inputType; - } - - public Class getJavaType() - { - return javaType; - } - - public String getXarName() - { - return xarName; - } - - @NotNull - public static PropertyType getFromURI(String concept, String datatype) - { - return getFromURI(concept, datatype, RESOURCE); - } - - @Deprecated // Eliminate this along with PropertyRow? Or at least combine with setValue() below. - abstract protected void init(PropertyRow row, Object value); - abstract protected void setValue(ObjectProperty property, Object value); - abstract protected Object getValue(ObjectProperty property); - public Object getPreviewValue(@Nullable String prefix) - { - return getValue(null); - } - - static - { - Map m = new HashMap<>(); - - for (PropertyType t : values()) - { - String uri = t.getTypeUri(); - m.put(uri, t); - m.put(t.getXmlName(), t); - - if (uri.startsWith("http://www.w3.org/2001/XMLSchema#") || uri.startsWith("http://www.labkey.org/exp/xml#")) - { - String xsdName = uri.substring(uri.indexOf('#') + 1); - m.put("xsd:" + xsdName, t); - m.put(xsdName, t); - } - } - - uriToProperty = m; - } - - public static PropertyType getFromURI(@Nullable String concept, String datatype, PropertyType def) - { - PropertyType p = uriToProperty.get(concept); - - if (null == p) - { - p = uriToProperty.get(datatype); - if (null == p) - p = def; - } - - return p; - } - - @NotNull - public static PropertyType getFromXarName(String xarName) - { - return getFromXarName(xarName, RESOURCE); - } - - public static PropertyType getFromXarName(String xarName, PropertyType def) - { - if (null == xarToProperty) - { - Map m = new CaseInsensitiveHashMap<>(); - for (PropertyType t : values()) - { - m.put(t.getXmlName(), t); - } - xarToProperty = m; - } - - PropertyType p = xarToProperty.get(xarName); - - return null == p ? def : p; - } - - public static PropertyType getFromClass(Class clazz) - { - if (clazz == BigDecimal.class) - clazz = Double.class; - - for (PropertyType t : values()) - { - if (t.javaType == null) - continue; - if (t.javaType.isAssignableFrom(clazz)) - return t; - } - - // after trying the primary types, we then try any additional types: - for (PropertyType t : values()) - { - if (t.additionalTypes == null || t.additionalTypes.length == 0) - continue; - for (Class type : t.additionalTypes) - { - if (type.isAssignableFrom(clazz)) - return t; - } - } - return PropertyType.STRING; - } - - @NotNull - public static PropertyType getFromJdbcType(JdbcType jdbcType) - { - return Objects.requireNonNull(getFromJdbcType(jdbcType, true)); - } - - @Nullable - public static PropertyType getFromJdbcType(JdbcType jdbcType, boolean throwIfNotFound) - { - for (PropertyType t : values()) - { - if (t.jdbcType.equals(jdbcType)) - return t; - } - if (throwIfNotFound) - throw new IllegalArgumentException("No such JdbcType mapping: " + (null != jdbcType ? jdbcType.getClass().toString() : "null")); - else - return null; - } - - @Nullable - public static PropertyType getFromJdbcTypeName(String typeName) - { - for (PropertyType t : values()) - { - if (typeName.equalsIgnoreCase(t.jdbcType.name())) - return t; - } - return null; - } - - public abstract SimpleTypeNames.Enum getXmlBeanType(); - - protected abstract Object convertExcelValue(Cell cell) throws ConversionException; - - public abstract Object convert(Object value) throws ConversionException; - - public static Object getFromExcelCell(Cell cell) throws ConversionException - { - if (ExcelFactory.isCellNumeric(cell)) - { - // Ugly, the POI implementation doesn't expose an explicit date type - if (org.apache.poi.ss.usermodel.DateUtil.isCellDateFormatted(cell)) - return DATE_TIME.convertExcelValue(cell); - else - // special handling for the "number type": prefer double. - // Without this, we'd default to integer - return DOUBLE.convertExcelValue(cell); - } - - for (PropertyType t : values()) - { - if (t.excelCellType == cell.getCellType()) - return t.convertExcelValue(cell); - } - return ExcelFactory.getCellStringValue(cell); - } - - public String getValueTypeColumn() - { - switch (this.getStorageType()) - { - case 's': - return "stringValue"; - case 'd': - return "dateTimeValue"; - case 'f': - return "floatValue"; - default: - throw new IllegalArgumentException("Unknown property type: " + this); - } - } -} +/* + * Copyright (c) 2008-2019 LabKey Corporation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.labkey.api.exp; + +import org.apache.commons.beanutils.ConversionException; +import org.apache.commons.beanutils.ConvertUtils; +import org.apache.poi.ss.usermodel.Cell; +import org.apache.poi.ss.usermodel.CellType; +import org.fhcrc.cpas.exp.xml.SimpleTypeNames; +import org.jetbrains.annotations.NotNull; +import org.jetbrains.annotations.Nullable; +import org.labkey.api.attachments.AttachmentFile; +import org.labkey.api.collections.CaseInsensitiveHashMap; +import org.labkey.api.data.JdbcType; +import org.labkey.api.data.MultiChoice; +import org.labkey.api.data.NameGenerator; +import org.labkey.api.exp.OntologyManager.PropertyRow; +import org.labkey.api.reader.ExcelFactory; +import org.labkey.api.util.DateUtil; +import org.labkey.vfs.FileLike; + +import java.io.File; +import java.math.BigDecimal; +import java.nio.ByteBuffer; +import java.sql.Array; +import java.sql.SQLException; +import java.sql.Time; +import java.text.DateFormat; +import java.text.ParseException; +import java.text.SimpleDateFormat; +import java.util.Date; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.TimeZone; + +import static org.labkey.api.util.IntegerUtils.asIntegerElseNull; +import static org.labkey.api.util.IntegerUtils.asLongElseNull; + +/** + * TODO: Add more types? Entity, Lsid, User, ... + */ +public enum PropertyType +{ + BOOLEAN("http://www.w3.org/2001/XMLSchema#boolean", "Boolean", 'f', JdbcType.BOOLEAN, 10, null, CellType.BOOLEAN, Boolean.class, Boolean.TYPE) + { + @Override + protected Object convertExcelValue(Cell cell) throws ConversionException + { + return cell.getBooleanCellValue(); + } + + @Override + public Object convert(Object value) throws ConversionException + { + boolean boolValue = false; + if (value instanceof Boolean) + boolValue = (Boolean)value; + else if (null != value && !"".equals(value)) + boolValue = (Boolean) ConvertUtils.convert(value.toString(), Boolean.class); + return boolValue; + } + + @Override + public SimpleTypeNames.Enum getXmlBeanType() + { + return SimpleTypeNames.BOOLEAN; + } + + @Override + protected void init(PropertyRow row, Object value) + { + Boolean b = (Boolean)value; + row.floatValue = b == Boolean.TRUE ? 1.0 : 0.0; + } + + @Override + protected void setValue(ObjectProperty property, Object value) + { + Boolean boolValue = null; + if (value instanceof Boolean) + boolValue = (Boolean)value; + else if (null != value) + boolValue = (Boolean) ConvertUtils.convert(value.toString(), Boolean.class); + property.floatValue = boolValue == null ? null : boolValue == Boolean.TRUE ? 1.0 : 0.0; + } + + @Override + protected Object getValue(ObjectProperty property) + { + return property.floatValue == null ? null : property.floatValue.intValue() != 0 ? Boolean.TRUE : Boolean.FALSE; + } + + @Override + public Object getPreviewValue(@Nullable String prefix) + { + return Boolean.TRUE; + } + }, + STRING("http://www.w3.org/2001/XMLSchema#string", "String", 's', JdbcType.VARCHAR, 4000, "text", CellType.STRING, String.class) + { + @Override + protected Object convertExcelValue(Cell cell) throws ConversionException + { + return cell.getStringCellValue(); + } + + @Override + public Object convert(Object value) throws ConversionException + { + if (value instanceof String) + return value; + else + return ConvertUtils.convert(value); + } + + @Override + public SimpleTypeNames.Enum getXmlBeanType() + { + return SimpleTypeNames.STRING; + } + + @Override + protected void init(PropertyRow row, Object value) + { + row.stringValue = (String)value; + } + + @Override + protected void setValue(ObjectProperty property, Object value) + { + property.stringValue = value == null ? null : value.toString(); + } + + @Override + protected Object getValue(ObjectProperty property) + { + return property.getStringValue(); + } + + @Override + public Object getPreviewValue(@Nullable String prefix) + { + return prefix + "Value"; + } + }, + // NOT an XMLSchema type uri??? + MULTI_LINE("http://www.w3.org/2001/XMLSchema#multiLine", "MultiLine", 's', JdbcType.VARCHAR, 4000, "textarea", CellType.STRING, String.class) + { + @Override + protected Object convertExcelValue(Cell cell) throws ConversionException + { + return cell.getStringCellValue(); + } + + @Override + public Object convert(Object value) throws ConversionException + { + if (value instanceof String) + return value; + else + return ConvertUtils.convert(value); + } + + @Override + public SimpleTypeNames.Enum getXmlBeanType() + { + return SimpleTypeNames.STRING; + } + + @Override + protected void init(PropertyRow row, Object value) + { + row.stringValue = (String)value; + } + + @Override + protected void setValue(ObjectProperty property, Object value) + { + property.stringValue = value == null ? null : value.toString(); + } + + @Override + protected Object getValue(ObjectProperty property) + { + return property.getStringValue(); + } + + @Override + public Object getPreviewValue(@Nullable String prefix) + { + return prefix + "Value"; + } + }, + MULTI_CHOICE("http://cpas.fhcrc.org/exp/xml#multiChoice", "MultiChoice", '?' /* unsupported in exp.PropertyValues */, JdbcType.ARRAY, 0, "textarea", CellType.STRING, List.class) + { + @Override + protected Object convertExcelValue(Cell cell) throws ConversionException + { + return ConvertUtils.convert(cell.getStringCellValue(), MultiChoice.Array.class); + } + + @Override + public Object convert(Object value) throws ConversionException + { + return MultiChoice.Converter.getInstance().convert(MultiChoice.Array.class, value); + } + + @Override + public SimpleTypeNames.Enum getXmlBeanType() + { + return SimpleTypeNames.STRING; + } + + @Override + protected void init(PropertyRow row, Object value) + { + throw new UnsupportedOperationException("TODO MultiChoice"); + } + + @Override + protected void setValue(ObjectProperty property, Object value) + { + if ((value instanceof java.sql.Array array)) + property.arrayValue = MultiChoice.Array.from(array); + else if (value != null) + property.arrayValue = MultiChoice.Array.from(new Object[]{value}); + } + + @Override + protected Object getValue(ObjectProperty property) + { + return property.arrayValue; + } + + @Override + public Object getPreviewValue(@Nullable String prefix) + { + return "Option 1, Option 2"; + } + }, + RESOURCE("http://www.w3.org/2000/01/rdf-schema#Resource", "PropertyURI", 's', JdbcType.VARCHAR, 4000, null, CellType.STRING, Identifiable.class) + { + @Override + protected Object convertExcelValue(Cell cell) throws ConversionException + { + return cell.getStringCellValue(); + } + + @Override + public Object convert(Object value) throws ConversionException + { + if (null == value) + return null; + if (value instanceof Identifiable) + return ((Identifiable) value).getLSID(); + else + return value.toString(); + } + + @Override + public SimpleTypeNames.Enum getXmlBeanType() + { + return SimpleTypeNames.STRING; + } + + @Override + protected void init(PropertyRow row, Object value) + { + row.stringValue = (String)value; + } + + @Override + protected void setValue(ObjectProperty property, Object value) + { + if (value instanceof Identifiable) + { + property.stringValue = ((Identifiable) value).getLSID(); + property.objectValue = (Identifiable) value; + } + else if (null != value) + property.stringValue = value.toString(); + } + + @Override + protected Object getValue(ObjectProperty property) + { + if (null != property.objectValue) + return property.objectValue; + else + return property.getStringValue(); + } + + @Override + public Object getPreviewValue(@Nullable String prefix) + { + return prefix + "Value"; + } + }, + INTEGER("http://www.w3.org/2001/XMLSchema#int", "Integer", 'f', JdbcType.INTEGER, 10, null, CellType.NUMERIC, Integer.class, Integer.TYPE, Long.class, Long.TYPE) + { + @Override + protected Object convertExcelValue(Cell cell) throws ConversionException + { + return (int)cell.getNumericCellValue(); + } + + @Override + public Object convert(Object value) throws ConversionException + { + if (null == value) + return null; + if (asIntegerElseNull(value) instanceof Integer i) + return i; + else + return ConvertUtils.convert(value.toString(), Integer.class); + } + + @Override + public SimpleTypeNames.Enum getXmlBeanType() + { + return SimpleTypeNames.INTEGER; + } + + @Override + protected void init(PropertyRow row, Object value) + { + Number n = (Number) value; + if (null != n) + row.floatValue = n.doubleValue(); + } + + @Override + protected void setValue(ObjectProperty property, Object value) + { + if (null == value) + property.floatValue = null; + else if (asIntegerElseNull(value) instanceof Integer i) + property.floatValue = i.doubleValue(); + else + property.floatValue = (Double) ConvertUtils.convert(value.toString(), Double.class); + } + + @Override + protected Object getValue(ObjectProperty property) + { + return property.floatValue == null ? null : property.floatValue.intValue(); + } + + @Override + public Object getPreviewValue(@Nullable String prefix) + { + return Integer.valueOf(3); + } + }, + BIGINT("http://www.w3.org/2001/XMLSchema#long", "Long", 'f', JdbcType.BIGINT, 10, null, CellType.NUMERIC, Long.class, Long.TYPE) + { + @Override + protected Object convertExcelValue(Cell cell) throws ConversionException + { + return (int)cell.getNumericCellValue(); + } + + @Override + public Object convert(Object value) throws ConversionException + { + if (null == value) + return null; + if (asLongElseNull(value) instanceof Long l) + return l; + else + return ConvertUtils.convert(value.toString(), Long.class); + } + + @Override + public SimpleTypeNames.Enum getXmlBeanType() + { + throw new UnsupportedOperationException(); + } + + @Override + protected void init(PropertyRow row, Object value) + { + Number n = (Number) value; + if (null != n) + row.floatValue = n.doubleValue(); + } + + @Override + protected void setValue(ObjectProperty property, Object value) + { + if (null == value) + property.floatValue = null; + else if (asLongElseNull(value) instanceof Long l) + property.floatValue = l.doubleValue(); + else + property.floatValue = (Double) ConvertUtils.convert(value.toString(), Double.class); + } + + @Override + protected Object getValue(ObjectProperty property) + { + return property.floatValue == null ? null : property.floatValue.longValue(); + } + + @Override + public Object getPreviewValue(@Nullable String prefix) + { + return Integer.valueOf(3); + } + }, + // NOT an XMLSchema type uri??? + BINARY("http://www.w3.org/2001/XMLSchema#binary", "Binary", 'f', JdbcType.BINARY, 10, null, CellType.NUMERIC, ByteBuffer.class) + { + @Override + protected Object convertExcelValue(Cell cell) throws ConversionException + { + return (int)cell.getNumericCellValue(); + } + + @Override + public Object convert(Object value) throws ConversionException + { + if (null == value) + return null; + if (value instanceof ByteBuffer) + return value; + else + return ConvertUtils.convert(value.toString(), ByteBuffer.class); + } + + @Override + public SimpleTypeNames.Enum getXmlBeanType() + { + throw new UnsupportedOperationException(); + } + + @Override + protected void init(PropertyRow row, Object value) + { + throw new UnsupportedOperationException(); + } + + @Override + protected void setValue(ObjectProperty property, Object value) + { + if (null != value) + property.floatValue = (Double) ConvertUtils.convert(value.toString(), Double.class); + } + + @Override + protected Object getValue(ObjectProperty property) + { + throw new UnsupportedOperationException(); + } + }, + /** Stored as a path to a file on the server's file system */ + FILE_LINK("http://cpas.fhcrc.org/exp/xml#fileLink", "FileLink", 's', JdbcType.VARCHAR, 400, "file", CellType.STRING, File.class) + { + @Override + protected Object convertExcelValue(Cell cell) throws ConversionException + { + return cell.getStringCellValue(); + } + + @Override + public Object convert(Object value) throws ConversionException + { + if (null == value) + return null; + if (value instanceof File) + return ((File) value).getPath(); + else + return String.valueOf(value); + } + + @Override + public SimpleTypeNames.Enum getXmlBeanType() + { + return SimpleTypeNames.FILE_LINK; + } + + @Override + protected void init(PropertyRow row, Object value) + { + row.stringValue = (String)value; + } + + @Override + protected void setValue(ObjectProperty property, Object value) + { + if (value instanceof File f) + property.stringValue = f.getPath(); + else if (value instanceof FileLike fl) + property.stringValue = fl.toNioPathForRead().toString(); + else + property.stringValue = value == null ? null : value.toString(); + } + + @Override + protected Object getValue(ObjectProperty property) + { + String value = property.getStringValue(); + return value == null ? null : new File(value); + } + + @Override + public Object getPreviewValue(@Nullable String prefix) + { + return prefix + "Value"; + } + }, + /** Stored in the database as a BLOB using AttachmentService */ + ATTACHMENT("http://www.labkey.org/exp/xml#attachment", "Attachment", 's', JdbcType.VARCHAR, 100, "file", CellType.STRING, File.class) + { + @Override + protected Object convertExcelValue(Cell cell) throws ConversionException + { + return cell.getStringCellValue(); + } + + @Override + public Object convert(Object value) throws ConversionException + { + if (null == value) + return null; + if (value instanceof File) + return ((File) value).getPath(); + else + return String.valueOf(value); + } + + @Override + public SimpleTypeNames.Enum getXmlBeanType() + { + throw new UnsupportedOperationException(); + } + + @Override + protected void init(PropertyRow row, Object value) + { + row.stringValue = (String)value; + } + + @Override + protected void setValue(ObjectProperty property, Object value) + { + if (value instanceof AttachmentFile) + { + property.stringValue = ((AttachmentFile)value).getFilename(); + } + else + property.stringValue = value == null ? null : value.toString(); + } + + @Override + protected Object getValue(ObjectProperty property) + { + return property.getStringValue(); + } + + @Override + public Object getPreviewValue(@Nullable String prefix) + { + return prefix + "Value"; + } + }, + DATE_TIME("http://www.w3.org/2001/XMLSchema#dateTime", "DateTime", 'd', JdbcType.TIMESTAMP, 100, null, CellType.NUMERIC, Date.class) + { + @Override + protected Object convertExcelValue(Cell cell) throws ConversionException + { + Date date = cell.getDateCellValue(); + if (date != null) + { + DateFormat format = new SimpleDateFormat("MM/dd/yyyy GG HH:mm:ss.SSS"); + format.setTimeZone(TimeZone.getDefault()); + String s = format.format(date); + try + { + date = format.parse(s); + } + catch (ParseException e) + { + throw new ConversionException(e); + } +// int offset = TimeZone.getDefault().getOffset(date.getTime()); +// date.setTime(date.getTime() - offset); + } + return date; + } + + @Override + public Object convert(Object value) throws ConversionException + { + if (null == value) + return null; + if (value instanceof Date) + return value; + else + { + String strVal = value.toString(); + if (DateUtil.isSignedDuration(strVal)) + strVal = JdbcType.TIMESTAMP.convert(value).toString(); + return ConvertUtils.convert(strVal, Date.class); + } + } + + @Override + public SimpleTypeNames.Enum getXmlBeanType() + { + return SimpleTypeNames.DATE_TIME; + } + + @Override + protected void init(PropertyRow row, Object value) + { + row.dateTimeValue = new java.sql.Time(((java.util.Date)value).getTime()); + } + + @Override + protected void setValue(ObjectProperty property, Object value) + { + if (value instanceof Date) + property.dateTimeValue = (Date) value; + else if (null != value) + property.dateTimeValue = (Date) ConvertUtils.convert(value.toString(), Date.class); + } + + @Override + protected Object getValue(ObjectProperty property) + { + return property.dateTimeValue; + } + + + @Override + public Object getPreviewValue(@Nullable String prefix) + { + return NameGenerator.PREVIEW_DATETIME_VALUE; + } + }, + DATE("http://www.w3.org/2001/XMLSchema#date", "Date", 'd', JdbcType.DATE, 100, null, CellType.NUMERIC, Date.class) + { + @Override + protected Object convertExcelValue(Cell cell) throws ConversionException + { + return DateUtil.getDateOnly((Date)DATE_TIME.convertExcelValue(cell)); + } + + @Override + public Object convert(Object value) throws ConversionException + { + return DateUtil.getDateOnly((Date)DATE_TIME.convert(value)); + } + + @Override + public SimpleTypeNames.Enum getXmlBeanType() + { + return SimpleTypeNames.DATE_TIME; + } + + @Override + protected void init(PropertyRow row, Object value) + { + row.dateTimeValue = new java.sql.Date(((java.util.Date)value).getTime()); + } + + @Override + protected void setValue(ObjectProperty property, Object value) + { + if (value instanceof Date) + property.dateTimeValue = (Date) value; + else if (null != value) + property.dateTimeValue = (Date) ConvertUtils.convert(value.toString(), Date.class); + } + + @Override + protected Object getValue(ObjectProperty property) + { + return property.dateTimeValue; + } + + @Override + public Object getPreviewValue(@Nullable String prefix) + { + return NameGenerator.PREVIEW_DATE_VALUE; + } + }, + TIME("http://www.w3.org/2001/XMLSchema#time", "Time", 'd', JdbcType.TIME, 100, null, CellType.NUMERIC, java.sql.Time.class) + { + @Override + protected Object convertExcelValue(Cell cell) throws ConversionException + { + return DateUtil.getTimeOnly((Date)DATE_TIME.convertExcelValue(cell)); + } + + @Override + public Object convert(Object value) throws ConversionException + { + if (null == value) + return null; + + if (value instanceof Time) + return value; + + if (value instanceof Date) + return DateUtil.getTimeOnly((Date) value); + + try + { + return ConvertUtils.convert(value, Time.class); + } + catch (Exception ignore) + { + } + + return DateUtil.getTimeOnly((Date)DATE_TIME.convert(value)); + } + + @Override + public SimpleTypeNames.Enum getXmlBeanType() + { + return SimpleTypeNames.DATE_TIME; + } + + @Override + protected void init(PropertyRow row, Object value) + { + row.dateTimeValue = new java.sql.Time(((java.util.Date)value).getTime()); + } + + @Override + protected void setValue(ObjectProperty property, Object value) + { + if (value instanceof Time) + property.dateTimeValue = (Time) value; + else if (null != value) + property.dateTimeValue = (Time) ConvertUtils.convert(value.toString(), Time.class); + } + + @Override + protected Object getValue(ObjectProperty property) + { + return property.dateTimeValue; + } + + @Override + public Object getPreviewValue(@Nullable String prefix) + { + return NameGenerator.PREVIEW_TIME_VALUE; + } + }, + DOUBLE("http://www.w3.org/2001/XMLSchema#double", "Double", 'f', JdbcType.DOUBLE, 20, null, CellType.NUMERIC, Double.class, Double.TYPE, Float.class, Float.TYPE) + { + @Override + protected Object convertExcelValue(Cell cell) throws ConversionException + { + return cell.getNumericCellValue(); + } + + @Override + public Object convert(Object value) throws ConversionException + { + if (null == value) + return null; + if (value instanceof Double) + return value; + else + return ConvertUtils.convert(String.valueOf(value), Double.class); + } + + @Override + public SimpleTypeNames.Enum getXmlBeanType() + { + return SimpleTypeNames.DOUBLE; + } + + @Override + protected void init(PropertyRow row, Object value) + { + Number n = (Number) value; + if (null != n) + row.floatValue = n.doubleValue(); + } + + @Override + protected void setValue(ObjectProperty property, Object value) + { + if (value instanceof Double) + property.floatValue = (Double) value; + else if (null != value) + property.floatValue = (Double) ConvertUtils.convert(value.toString(), Double.class); + } + + @Override + protected Object getValue(ObjectProperty property) + { + return property.floatValue; + } + + @Override + public Object getPreviewValue(@Nullable String prefix) + { + return 12.34; + } + }, + FLOAT("http://www.w3.org/2001/XMLSchema#float", "Float", 'f', JdbcType.REAL, 20, null, CellType.NUMERIC, Float.class, Float.TYPE) + { + @Override + protected Object convertExcelValue(Cell cell) throws ConversionException + { + return cell.getNumericCellValue(); + } + + @Override + public Object convert(Object value) throws ConversionException + { + if (null == value) + return null; + if (value instanceof Float) + return value; + else + return ConvertUtils.convert(String.valueOf(value), Float.class); + } + + @Override + public SimpleTypeNames.Enum getXmlBeanType() + { + throw new UnsupportedOperationException(); + } + + @Override + protected void init(PropertyRow row, Object value) + { + Number n = (Number) value; + if (null != n) + row.floatValue = n.doubleValue(); + } + + @Override + protected void setValue(ObjectProperty property, Object value) + { + if (value instanceof Double) + property.floatValue = (Double) value; + else if (null != value) + property.floatValue = (Double) ConvertUtils.convert(value.toString(), Double.class); + } + + @Override + protected Object getValue(ObjectProperty property) + { + return property.floatValue; + } + + @Override + public Object getPreviewValue(@Nullable String prefix) + { + return 12.34; + } + }, + DECIMAL("http://www.w3.org/2001/XMLSchema#decimal", "Decimal", 'f', JdbcType.DECIMAL, 20, null, CellType.NUMERIC, BigDecimal.class) + { + @Override + protected Object convertExcelValue(Cell cell) throws ConversionException + { + return cell.getNumericCellValue(); + } + + @Override + public Object convert(Object value) throws ConversionException + { + if (null == value) + return null; + if (value instanceof BigDecimal) + return value; + else + return ConvertUtils.convert(String.valueOf(value), BigDecimal.class); + } + + @Override + public SimpleTypeNames.Enum getXmlBeanType() + { + throw new UnsupportedOperationException(); + } + + @Override + protected void init(PropertyRow row, Object value) + { + Number n = (Number) value; + if (null != n) + row.floatValue = n.doubleValue(); + } + + @Override + protected void setValue(ObjectProperty property, Object value) + { + if (null != value) + property.floatValue = (Double) ConvertUtils.convert(value.toString(), Double.class); + } + + @Override + protected Object getValue(ObjectProperty property) + { + return property.floatValue; + } + + @Override + public Object getPreviewValue(@Nullable String prefix) + { + return 12.34; + } + }, + XML_TEXT("http://cpas.fhcrc.org/exp/xml#text-xml", "XmlText", 's', JdbcType.LONGVARCHAR, 4000, null, CellType.STRING, null) + { + @Override + protected Object convertExcelValue(Cell cell) throws ConversionException + { + return cell.getStringCellValue(); + } + + @Override + public Object convert(Object value) throws ConversionException + { + if (value instanceof String) + return value; + else + return ConvertUtils.convert(value); + } + + @Override + public SimpleTypeNames.Enum getXmlBeanType() + { + throw new UnsupportedOperationException(); + } + + @Override + protected void init(PropertyRow row, Object value) + { + throw new UnsupportedOperationException(); + } + + @Override + protected void setValue(ObjectProperty property, Object value) + { + throw new UnsupportedOperationException(); + } + + @Override + protected Object getValue(ObjectProperty property) + { + return property.getStringValue(); + } + + @Override + public Object getPreviewValue(@Nullable String prefix) + { + return prefix + "Value"; + } + }; + + private final String typeURI; + private final String xarName; + private final char storageType; + private final CellType excelCellType; + private final @NotNull JdbcType jdbcType; + private final int scale; + private final String inputType; + private final Class javaType; + private final Class[] additionalTypes; + + private static Map uriToProperty; + private static Map xarToProperty = null; + + PropertyType(String typeURI, + String xarName, + char storageType, + @NotNull JdbcType jdbcType, + int scale, + String inputType, + CellType excelCellType, + Class javaType, + Class... additionalTypes) + { + this.typeURI = typeURI; + this.xarName = xarName; + this.storageType = storageType; + this.jdbcType = jdbcType; + this.scale = scale; + this.inputType = inputType; + this.javaType = javaType; + this.excelCellType = excelCellType; + this.additionalTypes = additionalTypes; + } + + public String getTypeUri() + { + return typeURI; + } + + public String getXmlName() + { + return xarName; + } + + public char getStorageType() + { + return storageType; + } + + @NotNull + public JdbcType getJdbcType() + { + return jdbcType; + } + + public int getScale() + { + return scale; + } + + @Nullable + public String getInputType() + { + return inputType; + } + + public Class getJavaType() + { + return javaType; + } + + public String getXarName() + { + return xarName; + } + + @NotNull + public static PropertyType getFromURI(String concept, String datatype) + { + return getFromURI(concept, datatype, RESOURCE); + } + + @Deprecated // Eliminate this along with PropertyRow? Or at least combine with setValue() below. + abstract protected void init(PropertyRow row, Object value); + abstract protected void setValue(ObjectProperty property, Object value); + abstract protected Object getValue(ObjectProperty property); + public Object getPreviewValue(@Nullable String prefix) + { + return getValue(null); + } + + static + { + Map m = new HashMap<>(); + + for (PropertyType t : values()) + { + String uri = t.getTypeUri(); + m.put(uri, t); + m.put(t.getXmlName(), t); + + if (uri.startsWith("http://www.w3.org/2001/XMLSchema#") || uri.startsWith("http://www.labkey.org/exp/xml#")) + { + String xsdName = uri.substring(uri.indexOf('#') + 1); + m.put("xsd:" + xsdName, t); + m.put(xsdName, t); + } + } + + uriToProperty = m; + } + + public static PropertyType getFromURI(@Nullable String concept, String datatype, PropertyType def) + { + PropertyType p = uriToProperty.get(concept); + + if (null == p) + { + p = uriToProperty.get(datatype); + if (null == p) + p = def; + } + + return p; + } + + @NotNull + public static PropertyType getFromXarName(String xarName) + { + return getFromXarName(xarName, RESOURCE); + } + + public static PropertyType getFromXarName(String xarName, PropertyType def) + { + if (null == xarToProperty) + { + Map m = new CaseInsensitiveHashMap<>(); + for (PropertyType t : values()) + { + m.put(t.getXmlName(), t); + } + xarToProperty = m; + } + + PropertyType p = xarToProperty.get(xarName); + + return null == p ? def : p; + } + + public static PropertyType getFromClass(Class clazz) + { + if (clazz == BigDecimal.class) + clazz = Double.class; + + for (PropertyType t : values()) + { + if (t.javaType == null) + continue; + if (t.javaType.isAssignableFrom(clazz)) + return t; + } + + // after trying the primary types, we then try any additional types: + for (PropertyType t : values()) + { + if (t.additionalTypes == null || t.additionalTypes.length == 0) + continue; + for (Class type : t.additionalTypes) + { + if (type.isAssignableFrom(clazz)) + return t; + } + } + return PropertyType.STRING; + } + + @NotNull + public static PropertyType getFromJdbcType(JdbcType jdbcType) + { + return Objects.requireNonNull(getFromJdbcType(jdbcType, true)); + } + + @Nullable + public static PropertyType getFromJdbcType(JdbcType jdbcType, boolean throwIfNotFound) + { + for (PropertyType t : values()) + { + if (t.jdbcType.equals(jdbcType)) + return t; + } + if (throwIfNotFound) + throw new IllegalArgumentException("No such JdbcType mapping: " + (null != jdbcType ? jdbcType.getClass().toString() : "null")); + else + return null; + } + + @Nullable + public static PropertyType getFromJdbcTypeName(String typeName) + { + for (PropertyType t : values()) + { + if (typeName.equalsIgnoreCase(t.jdbcType.name())) + return t; + } + return null; + } + + public abstract SimpleTypeNames.Enum getXmlBeanType(); + + protected abstract Object convertExcelValue(Cell cell) throws ConversionException; + + public abstract Object convert(Object value) throws ConversionException; + + public static Object getFromExcelCell(Cell cell) throws ConversionException + { + if (ExcelFactory.isCellNumeric(cell)) + { + // Ugly, the POI implementation doesn't expose an explicit date type + if (org.apache.poi.ss.usermodel.DateUtil.isCellDateFormatted(cell)) + return DATE_TIME.convertExcelValue(cell); + else + // special handling for the "number type": prefer double. + // Without this, we'd default to integer + return DOUBLE.convertExcelValue(cell); + } + + for (PropertyType t : values()) + { + if (t.excelCellType == cell.getCellType()) + return t.convertExcelValue(cell); + } + return ExcelFactory.getCellStringValue(cell); + } + + public String getValueTypeColumn() + { + switch (this.getStorageType()) + { + case 's': + return "stringValue"; + case 'd': + return "dateTimeValue"; + case 'f': + return "floatValue"; + default: + throw new IllegalArgumentException("Unknown property type: " + this); + } + } +} diff --git a/api/src/org/labkey/api/exp/property/DomainKind.java b/api/src/org/labkey/api/exp/property/DomainKind.java index 65bb8a58ff2..1de58f1c673 100644 --- a/api/src/org/labkey/api/exp/property/DomainKind.java +++ b/api/src/org/labkey/api/exp/property/DomainKind.java @@ -1,439 +1,439 @@ -/* - * Copyright (c) 2008-2019 LabKey Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.labkey.api.exp.property; - -import org.apache.logging.log4j.Logger; -import org.jetbrains.annotations.NotNull; -import org.jetbrains.annotations.Nullable; -import org.labkey.api.collections.CaseInsensitiveHashSet; -import org.labkey.api.data.Container; -import org.labkey.api.data.ContainerFilter; -import org.labkey.api.data.DbSchemaType; -import org.labkey.api.data.DbScope; -import org.labkey.api.data.NameExpressionValidationResult; -import org.labkey.api.data.PropertyStorageSpec; -import org.labkey.api.data.SQLFragment; -import org.labkey.api.data.SchemaTableInfo; -import org.labkey.api.data.TableInfo; -import org.labkey.api.data.UpdateableTableInfo; -import org.labkey.api.exp.Handler; -import org.labkey.api.exp.PropertyDescriptor; -import org.labkey.api.exp.TemplateInfo; -import org.labkey.api.gwt.client.DefaultValueType; -import org.labkey.api.gwt.client.model.GWTDomain; -import org.labkey.api.gwt.client.model.GWTPropertyDescriptor; -import org.labkey.api.query.UserSchema; -import org.labkey.api.query.ValidationException; -import org.labkey.api.security.User; -import org.labkey.api.security.UserPrincipal; -import org.labkey.api.security.permissions.Permission; -import org.labkey.api.util.logging.LogHelper; -import org.labkey.api.view.ActionURL; -import org.labkey.api.view.NavTree; -import org.labkey.api.view.UnauthorizedException; -import org.labkey.api.writer.ContainerUser; -import org.labkey.data.xml.domainTemplate.DomainTemplateType; - -import java.util.Collections; -import java.util.LinkedHashSet; -import java.util.List; -import java.util.Map; -import java.util.Set; -import java.util.stream.Collectors; - -abstract public class DomainKind implements Handler -{ - public static final Logger LOG = LogHelper.getLogger(DomainKind.class, "Generic domain kind activities."); - abstract public String getKindName(); - - /** - * Return a class of DomainKind's bean which carries domain specific properties. - * This class will be used when marshalling/unmarshalling via Jackson during Create and Save/Update Domain - * @return Class of DomainKind's bean with domain specific properties - */ - abstract public Class getTypeClass(); - - /** - * Give the domain kind a chance to process / map the arguments before converting it to the DomainKind properties object. - * @param arguments initial arguments coming from the caller - * @return updated arguments - */ - public Map processArguments(Container container, User user, Map arguments) - { - return arguments; - } - - abstract public String getTypeLabel(Domain domain); - abstract public SQLFragment sqlObjectIdsInDomain(Domain domain); - - /** - * Create a DomainURI for a Domain that may or may not exist yet. - */ - abstract public String generateDomainURI(String schemaName, String queryName, Container container, User user); - - abstract public @Nullable ActionURL urlShowData(Domain domain, ContainerUser containerUser); - abstract public @Nullable ActionURL urlEditDefinition(Domain domain, ContainerUser containerUser); - abstract public ActionURL urlCreateDefinition(String schemaName, String queryName, Container container, User user); - - // Override to return a non-null String and the generic domain editor will display it in an "Instructions" webpart above the field properties - public @Nullable String getDomainEditorInstructions() - { - return null; - } - - abstract public boolean canCreateDefinition(User user, Container container); - - abstract public boolean canEditDefinition(User user, Domain domain); - - abstract public boolean canDeleteDefinition(User user, Domain domain); - - // Override to customize the nav trail on shared pages like edit domain - abstract public void addNavTrail(NavTree root, Container c, User user); - - // Do any special handling before a PropertyDescriptor is deleted -- do nothing by default - abstract public void deletePropertyDescriptor(Domain domain, User user, PropertyDescriptor pd); - - /** - * Return the set of names that should not be allowed for properties. E.g. - * the names of columns from the hard table underlying this type - * @return set of strings containing the names. This will be compared ignoring case - */ - abstract public Set getReservedPropertyNames(Domain domain, User user); - - public Set getReservedPropertyNames(Domain domain, User user, boolean forCreate) - { - return getReservedPropertyNames(domain, user); - } - - public Set getReservedPropertyNamePrefixes() - { - return Collections.emptySet(); - } - - /** - * Return the set of names that are always required and cannot subsequently - * be deleted. - * @return set of strings containing the names. This will be compared ignoring case - */ - abstract public Set getMandatoryPropertyNames(Domain domain); - - // CONSIDER: have DomainKind supply and IDomainInstance or similar - // so that it can hold instance data (e.g. a DatasetDefinition) - - /** - * Get DomainKind specific properties. - * @param domain The domain design. - * @param container Container - * @param user User - * @return Return object that holds DomainKind specific properties. - */ - abstract public @Nullable T getDomainKindProperties(GWTDomain domain, Container container, User user); - - /** - * Create a Domain appropriate for this DomainKind. - * - * @param domain The domain design. - * @param options Any domain kind specific properties/options. - * @param container Container - * @param user User - * @param forUpdate Whether the returned domain should be mutable or not - * @return The newly created Domain. - */ - abstract public Domain createDomain(GWTDomain domain, T options, Container container, User user, @Nullable TemplateInfo templateInfo, boolean forUpdate); - - /** - * Update a Domain definition appropriate for this DomainKind. - * @param original The original domain definition. - * @param update The updated domain definition. - * @param options Any domain kind specific properties/options. - * @param container Container - * @param user User - * @return A list of errors collected during the update. - */ - abstract public ValidationException updateDomain(GWTDomain original, GWTDomain update, - @Nullable T options, Container container, User user, boolean includeWarnings, @Nullable String auditUserComment); - /** - * Delete a Domain and its associated data. - * @param domain The domain to delete - */ - abstract public void deleteDomain(User user, Domain domain, @Nullable String auditUserComment); - - /** - * Get base properties defined for that domainkind. The domain parameter is only when there may be a condition - * with the particular domain that could affect the base properties (see DatasetDomainKind). Other domainkinds - * may pass through null (see AssayDomainKind). - */ - abstract public Set getBaseProperties(@Nullable Domain domain); - - /** - * Any additional properties which will get special handling in the Properties Editor. - * First use case is Lists get their property-backed primary key field added to protect it from imports and - * exclude it from exports - */ - public Set getAdditionalProtectedProperties(Domain domain) - { - return Collections.emptySet(); - } - - public Set getAdditionalProtectedPropertyNames(Domain domain) - { - Set properties = new LinkedHashSet<>(); - for (PropertyStorageSpec pss : getAdditionalProtectedProperties(domain)) - properties.add(pss.getName()); - return properties; - } - - public abstract Set getPropertyForeignKeys(Container container); - - /** - * If domains of this kind should get hard tables automatically provisioned, this returns - * the db schema where they reside. If it is null, hard tables are not to be provisioned for domains of this kind. - */ - abstract public DbScope getScope(); - abstract public String getStorageSchemaName(); - public boolean isProvisioned(Container container, String name) - { - return getStorageSchemaName() != null; - } - abstract public Set getPropertyIndices(Domain domain); - - /** - * If domain needs metadata, give the metadata schema and table names - */ - abstract public String getMetaDataSchemaName(); - abstract public String getMetaDataTableName(); - - /** - * Determines if the domain has any existing rows where the value is null for the given property - * Perhaps DomainKind should have getTableInfo() method. - */ - abstract public boolean hasNullValues(Domain domain, DomainProperty prop); - - public DbSchemaType getSchemaType() - { - return DbSchemaType.Provisioned; - } - - /** ask the domain to clear caches related to this domain */ - public void invalidate(Domain domain) - { - String schemaName = getStorageSchemaName(); - if (null == schemaName) - return; - - String storageTableName = domain.getStorageTableName(); - - if (null != storageTableName) - { - LOG.debug("Invalidating " + schemaName + "." + storageTableName); - getScope().invalidateTable(schemaName, storageTableName, getSchemaType()); - } - else - { - LOG.debug("Invalidating " + schemaName); - getScope().invalidateSchema(schemaName, getSchemaType()); - } - } - - /** - * Set of hard table names in this schema that are not provision tables - */ - abstract public Set getNonProvisionedTableNames(); - - abstract public PropertyStorageSpec getPropertySpec(PropertyDescriptor pd, Domain domain); - - /** - * @return true if we created property descriptors for base properties in the domain - */ - public boolean hasPropertiesIncludeBaseProperties() - { - return false; - } - - /** - * Default for all domain kinds is to not delete data. Lists and Datasets override this. - */ - public boolean isDeleteAllDataOnFieldImport() - { - return false; - } - - public @Nullable TableInfo getTableInfo(User user, Container container, String name, @Nullable ContainerFilter cf) - { - return null; - } - - public @Nullable TableInfo getTableInfo(User user, Container container, Domain domain, @Nullable ContainerFilter cf) - { - return getTableInfo(user, container, domain.getName(), cf); - } - - /** Called for provisioned tables after StorageProvisioner has loaded them from JDBC but before they are locked and - * cached. Use this to decorate the SchemaTableInfo with additional meta data, for example. - * - * NOTE: this is the raw-cached SchemaTableInfo, some column names may not match expected property names - * see PropertyDescriptor.getName(), PropertyDescriptor.getStorageColumnName() - */ - public void afterLoadTable(SchemaTableInfo ti, Domain domain) - { - // Most DomainKinds do nothing here - } - - /** - * Check if existing string data fits in property scale - * @param domain to execute within - * @param prop property to check - * @return true if the DomainProperty is a string and a value exists that is greater than the DomainProperty's max length - */ - public boolean exceedsMaxLength(Domain domain, DomainProperty prop) - { - //Most domains don't need to do anything here - return false; - } - - public boolean ensurePropertyLookup() - { - return false; - } - - /** - * @return true if template type in .template.xml is an instance of a module specific TemplateType pojo (which is created when - * domainTemplate.xsd is processed resulting in auto-generated TemplateType classes) - */ - public boolean matchesTemplateXML(String templateName, DomainTemplateType template, List properties) - { - return false; - } - - public boolean allowFileLinkProperties() { return false; } - public boolean allowAttachmentProperties() { return false; } - public boolean allowFlagProperties() { return true; } - public boolean allowTextChoiceProperties() { return true; } - public boolean allowMultiChoiceProperties() { return false; } - public boolean allowSampleSubjectProperties() { return true; } - public boolean allowTimepointProperties() { return false; } - public boolean allowUniqueConstraintProperties() { return false; } - public boolean allowCalculatedFields() { return false; } - public boolean showDefaultValueSettings() { return false; } - - public List getDisabledSystemFields(List disabledSystemFields) - { - if (disabledSystemFields == null || disabledSystemFields.isEmpty()) - return disabledSystemFields; - - Set nonDisablebleFields = getNonDisablebleFields(); - if (nonDisablebleFields == null || nonDisablebleFields.isEmpty()) - return disabledSystemFields; - - final Set disallowedFields = new CaseInsensitiveHashSet(nonDisablebleFields); - - return disabledSystemFields.stream().filter(f -> !disallowedFields.contains(f)).collect(Collectors.toList()); - } - - protected Set getNonDisablebleFields() - { - return null; - } - - public DefaultValueType[] getDefaultValueOptions(Domain domain) - { - return new DefaultValueType[] { DefaultValueType.FIXED_EDITABLE, DefaultValueType.LAST_ENTERED }; - } - - public DefaultValueType getDefaultDefaultType(Domain domain) - { - return DefaultValueType.FIXED_EDITABLE; - } - - public String getObjectUriColumnName() - { - return null; - } - - public UpdateableTableInfo.ObjectUriType getObjectUriColumn() - { - return null; - } - - /** - * Overridable validity check for domain name. Base implementation does nothing. - */ - public void validateDomainName(Container container, User user, @Nullable Domain domain, String name) - {} - - /** - * Overridable validity check. Base only executes canCreateDefinition check. - * NOTE: Due to historical limitations throws runtime exceptions instead of validation errors - * @param container being executed upon - * @param user executing service call - * @param options map to check - * @param name of design - * @param domain the existing domain object for the create/save action - * @param updatedDomainDesign the updated domain design being sent for the create/save action - */ - public void validateOptions(Container container, User user, T options, String name, Domain domain, GWTDomain updatedDomainDesign) - { - boolean isUpdate = domain != null; - - if (!isUpdate && !this.canCreateDefinition(user, container)) - throw new UnauthorizedException("You don't have permission to create a new domain"); - - if (isUpdate && !canEditDefinition(user, domain)) - throw new UnauthorizedException("You don't have permission to edit this domain"); - } - - public NameExpressionValidationResult validateNameExpressions(T options, GWTDomain domainDesign, Container container) - { - return null; - } - - /** - * @return Return preview name(s) based on the name expression configured for the designer. For DataClass, - * up to one preview names is returned. For samples, up to 2 names can be returned, with the 1st one being - * the sample preview name and the 2nd being the aliquot preview name. - */ - public @Nullable List getDomainNamePreviews(String schemaName, String queryName, Container container, User user) - { - return null; - } - - public boolean hasPermission(@NotNull UserPrincipal user, @NotNull Class perm, @NotNull UserSchema userSchema) - { - return true; - } - - // Is this domain kind considered "user managed"? e.g. list, sample type, etc - public boolean isUserCreatedType() - { - return true; - } - - public boolean supportsPhiLevel() - { - return false; - } - - public boolean supportsNamingPattern() - { - return false; - } - - public String getDomainFileDirectory() - { - return getKindName(); - } -} +/* + * Copyright (c) 2008-2019 LabKey Corporation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.labkey.api.exp.property; + +import org.apache.logging.log4j.Logger; +import org.jetbrains.annotations.NotNull; +import org.jetbrains.annotations.Nullable; +import org.labkey.api.collections.CaseInsensitiveHashSet; +import org.labkey.api.data.Container; +import org.labkey.api.data.ContainerFilter; +import org.labkey.api.data.DbSchemaType; +import org.labkey.api.data.DbScope; +import org.labkey.api.data.NameExpressionValidationResult; +import org.labkey.api.data.PropertyStorageSpec; +import org.labkey.api.data.SQLFragment; +import org.labkey.api.data.SchemaTableInfo; +import org.labkey.api.data.TableInfo; +import org.labkey.api.data.UpdateableTableInfo; +import org.labkey.api.exp.Handler; +import org.labkey.api.exp.PropertyDescriptor; +import org.labkey.api.exp.TemplateInfo; +import org.labkey.api.gwt.client.DefaultValueType; +import org.labkey.api.gwt.client.model.GWTDomain; +import org.labkey.api.gwt.client.model.GWTPropertyDescriptor; +import org.labkey.api.query.UserSchema; +import org.labkey.api.query.ValidationException; +import org.labkey.api.security.User; +import org.labkey.api.security.UserPrincipal; +import org.labkey.api.security.permissions.Permission; +import org.labkey.api.util.logging.LogHelper; +import org.labkey.api.view.ActionURL; +import org.labkey.api.view.NavTree; +import org.labkey.api.view.UnauthorizedException; +import org.labkey.api.writer.ContainerUser; +import org.labkey.data.xml.domainTemplate.DomainTemplateType; + +import java.util.Collections; +import java.util.LinkedHashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.stream.Collectors; + +abstract public class DomainKind implements Handler +{ + public static final Logger LOG = LogHelper.getLogger(DomainKind.class, "Generic domain kind activities."); + abstract public String getKindName(); + + /** + * Return a class of DomainKind's bean which carries domain specific properties. + * This class will be used when marshalling/unmarshalling via Jackson during Create and Save/Update Domain + * @return Class of DomainKind's bean with domain specific properties + */ + abstract public Class getTypeClass(); + + /** + * Give the domain kind a chance to process / map the arguments before converting it to the DomainKind properties object. + * @param arguments initial arguments coming from the caller + * @return updated arguments + */ + public Map processArguments(Container container, User user, Map arguments) + { + return arguments; + } + + abstract public String getTypeLabel(Domain domain); + abstract public SQLFragment sqlObjectIdsInDomain(Domain domain); + + /** + * Create a DomainURI for a Domain that may or may not exist yet. + */ + abstract public String generateDomainURI(String schemaName, String queryName, Container container, User user); + + abstract public @Nullable ActionURL urlShowData(Domain domain, ContainerUser containerUser); + abstract public @Nullable ActionURL urlEditDefinition(Domain domain, ContainerUser containerUser); + abstract public ActionURL urlCreateDefinition(String schemaName, String queryName, Container container, User user); + + // Override to return a non-null String and the generic domain editor will display it in an "Instructions" webpart above the field properties + public @Nullable String getDomainEditorInstructions() + { + return null; + } + + abstract public boolean canCreateDefinition(User user, Container container); + + abstract public boolean canEditDefinition(User user, Domain domain); + + abstract public boolean canDeleteDefinition(User user, Domain domain); + + // Override to customize the nav trail on shared pages like edit domain + abstract public void addNavTrail(NavTree root, Container c, User user); + + // Do any special handling before a PropertyDescriptor is deleted -- do nothing by default + abstract public void deletePropertyDescriptor(Domain domain, User user, PropertyDescriptor pd); + + /** + * Return the set of names that should not be allowed for properties. E.g. + * the names of columns from the hard table underlying this type + * @return set of strings containing the names. This will be compared ignoring case + */ + abstract public Set getReservedPropertyNames(Domain domain, User user); + + public Set getReservedPropertyNames(Domain domain, User user, boolean forCreate) + { + return getReservedPropertyNames(domain, user); + } + + public Set getReservedPropertyNamePrefixes() + { + return Collections.emptySet(); + } + + /** + * Return the set of names that are always required and cannot subsequently + * be deleted. + * @return set of strings containing the names. This will be compared ignoring case + */ + abstract public Set getMandatoryPropertyNames(Domain domain); + + // CONSIDER: have DomainKind supply and IDomainInstance or similar + // so that it can hold instance data (e.g. a DatasetDefinition) + + /** + * Get DomainKind specific properties. + * @param domain The domain design. + * @param container Container + * @param user User + * @return Return object that holds DomainKind specific properties. + */ + abstract public @Nullable T getDomainKindProperties(GWTDomain domain, Container container, User user); + + /** + * Create a Domain appropriate for this DomainKind. + * + * @param domain The domain design. + * @param options Any domain kind specific properties/options. + * @param container Container + * @param user User + * @param forUpdate Whether the returned domain should be mutable or not + * @return The newly created Domain. + */ + abstract public Domain createDomain(GWTDomain domain, T options, Container container, User user, @Nullable TemplateInfo templateInfo, boolean forUpdate); + + /** + * Update a Domain definition appropriate for this DomainKind. + * @param original The original domain definition. + * @param update The updated domain definition. + * @param options Any domain kind specific properties/options. + * @param container Container + * @param user User + * @return A list of errors collected during the update. + */ + abstract public ValidationException updateDomain(GWTDomain original, GWTDomain update, + @Nullable T options, Container container, User user, boolean includeWarnings, @Nullable String auditUserComment); + /** + * Delete a Domain and its associated data. + * @param domain The domain to delete + */ + abstract public void deleteDomain(User user, Domain domain, @Nullable String auditUserComment); + + /** + * Get base properties defined for that domainkind. The domain parameter is only when there may be a condition + * with the particular domain that could affect the base properties (see DatasetDomainKind). Other domainkinds + * may pass through null (see AssayDomainKind). + */ + abstract public Set getBaseProperties(@Nullable Domain domain); + + /** + * Any additional properties which will get special handling in the Properties Editor. + * First use case is Lists get their property-backed primary key field added to protect it from imports and + * exclude it from exports + */ + public Set getAdditionalProtectedProperties(Domain domain) + { + return Collections.emptySet(); + } + + public Set getAdditionalProtectedPropertyNames(Domain domain) + { + Set properties = new LinkedHashSet<>(); + for (PropertyStorageSpec pss : getAdditionalProtectedProperties(domain)) + properties.add(pss.getName()); + return properties; + } + + public abstract Set getPropertyForeignKeys(Container container); + + /** + * If domains of this kind should get hard tables automatically provisioned, this returns + * the db schema where they reside. If it is null, hard tables are not to be provisioned for domains of this kind. + */ + abstract public DbScope getScope(); + abstract public String getStorageSchemaName(); + public boolean isProvisioned(Container container, String name) + { + return getStorageSchemaName() != null; + } + abstract public Set getPropertyIndices(Domain domain); + + /** + * If domain needs metadata, give the metadata schema and table names + */ + abstract public String getMetaDataSchemaName(); + abstract public String getMetaDataTableName(); + + /** + * Determines if the domain has any existing rows where the value is null for the given property + * Perhaps DomainKind should have getTableInfo() method. + */ + abstract public boolean hasNullValues(Domain domain, DomainProperty prop); + + public DbSchemaType getSchemaType() + { + return DbSchemaType.Provisioned; + } + + /** ask the domain to clear caches related to this domain */ + public void invalidate(Domain domain) + { + String schemaName = getStorageSchemaName(); + if (null == schemaName) + return; + + String storageTableName = domain.getStorageTableName(); + + if (null != storageTableName) + { + LOG.debug("Invalidating " + schemaName + "." + storageTableName); + getScope().invalidateTable(schemaName, storageTableName, getSchemaType()); + } + else + { + LOG.debug("Invalidating " + schemaName); + getScope().invalidateSchema(schemaName, getSchemaType()); + } + } + + /** + * Set of hard table names in this schema that are not provision tables + */ + abstract public Set getNonProvisionedTableNames(); + + abstract public PropertyStorageSpec getPropertySpec(PropertyDescriptor pd, Domain domain); + + /** + * @return true if we created property descriptors for base properties in the domain + */ + public boolean hasPropertiesIncludeBaseProperties() + { + return false; + } + + /** + * Default for all domain kinds is to not delete data. Lists and Datasets override this. + */ + public boolean isDeleteAllDataOnFieldImport() + { + return false; + } + + public @Nullable TableInfo getTableInfo(User user, Container container, String name, @Nullable ContainerFilter cf) + { + return null; + } + + public @Nullable TableInfo getTableInfo(User user, Container container, Domain domain, @Nullable ContainerFilter cf) + { + return getTableInfo(user, container, domain.getName(), cf); + } + + /** Called for provisioned tables after StorageProvisioner has loaded them from JDBC but before they are locked and + * cached. Use this to decorate the SchemaTableInfo with additional meta data, for example. + * + * NOTE: this is the raw-cached SchemaTableInfo, some column names may not match expected property names + * see PropertyDescriptor.getName(), PropertyDescriptor.getStorageColumnName() + */ + public void afterLoadTable(SchemaTableInfo ti, Domain domain) + { + // Most DomainKinds do nothing here + } + + /** + * Check if existing string data fits in property scale + * @param domain to execute within + * @param prop property to check + * @return true if the DomainProperty is a string and a value exists that is greater than the DomainProperty's max length + */ + public boolean exceedsMaxLength(Domain domain, DomainProperty prop) + { + //Most domains don't need to do anything here + return false; + } + + public boolean ensurePropertyLookup() + { + return false; + } + + /** + * @return true if template type in .template.xml is an instance of a module specific TemplateType pojo (which is created when + * domainTemplate.xsd is processed resulting in auto-generated TemplateType classes) + */ + public boolean matchesTemplateXML(String templateName, DomainTemplateType template, List properties) + { + return false; + } + + public boolean allowFileLinkProperties() { return false; } + public boolean allowAttachmentProperties() { return false; } + public boolean allowFlagProperties() { return true; } + public boolean allowTextChoiceProperties() { return true; } + public boolean allowMultiChoiceProperties() { return false; } + public boolean allowSampleSubjectProperties() { return true; } + public boolean allowTimepointProperties() { return false; } + public boolean allowUniqueConstraintProperties() { return false; } + public boolean allowCalculatedFields() { return false; } + public boolean showDefaultValueSettings() { return false; } + + public List getDisabledSystemFields(List disabledSystemFields) + { + if (disabledSystemFields == null || disabledSystemFields.isEmpty()) + return disabledSystemFields; + + Set nonDisablebleFields = getNonDisablebleFields(); + if (nonDisablebleFields == null || nonDisablebleFields.isEmpty()) + return disabledSystemFields; + + final Set disallowedFields = new CaseInsensitiveHashSet(nonDisablebleFields); + + return disabledSystemFields.stream().filter(f -> !disallowedFields.contains(f)).collect(Collectors.toList()); + } + + protected Set getNonDisablebleFields() + { + return null; + } + + public DefaultValueType[] getDefaultValueOptions(Domain domain) + { + return new DefaultValueType[] { DefaultValueType.FIXED_EDITABLE, DefaultValueType.LAST_ENTERED }; + } + + public DefaultValueType getDefaultDefaultType(Domain domain) + { + return DefaultValueType.FIXED_EDITABLE; + } + + public String getObjectUriColumnName() + { + return null; + } + + public UpdateableTableInfo.ObjectUriType getObjectUriColumn() + { + return null; + } + + /** + * Overridable validity check for domain name. Base implementation does nothing. + */ + public void validateDomainName(Container container, User user, @Nullable Domain domain, String name) + {} + + /** + * Overridable validity check. Base only executes canCreateDefinition check. + * NOTE: Due to historical limitations throws runtime exceptions instead of validation errors + * @param container being executed upon + * @param user executing service call + * @param options map to check + * @param name of design + * @param domain the existing domain object for the create/save action + * @param updatedDomainDesign the updated domain design being sent for the create/save action + */ + public void validateOptions(Container container, User user, T options, String name, Domain domain, GWTDomain updatedDomainDesign) + { + boolean isUpdate = domain != null; + + if (!isUpdate && !this.canCreateDefinition(user, container)) + throw new UnauthorizedException("You don't have permission to create a new domain"); + + if (isUpdate && !canEditDefinition(user, domain)) + throw new UnauthorizedException("You don't have permission to edit this domain"); + } + + public NameExpressionValidationResult validateNameExpressions(T options, GWTDomain domainDesign, Container container) + { + return null; + } + + /** + * @return Return preview name(s) based on the name expression configured for the designer. For DataClass, + * up to one preview names is returned. For samples, up to 2 names can be returned, with the 1st one being + * the sample preview name and the 2nd being the aliquot preview name. + */ + public @Nullable List getDomainNamePreviews(String schemaName, String queryName, Container container, User user) + { + return null; + } + + public boolean hasPermission(@NotNull UserPrincipal user, @NotNull Class perm, @NotNull UserSchema userSchema) + { + return true; + } + + // Is this domain kind considered "user managed"? e.g. list, sample type, etc + public boolean isUserCreatedType() + { + return true; + } + + public boolean supportsPhiLevel() + { + return false; + } + + public boolean supportsNamingPattern() + { + return false; + } + + public String getDomainFileDirectory() + { + return getKindName(); + } +} diff --git a/api/src/org/labkey/api/settings/AppProps.java b/api/src/org/labkey/api/settings/AppProps.java index 0ed0dc058c3..ba493df5f8d 100644 --- a/api/src/org/labkey/api/settings/AppProps.java +++ b/api/src/org/labkey/api/settings/AppProps.java @@ -1,268 +1,268 @@ -/* - * Copyright (c) 2008-2019 LabKey Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.labkey.api.settings; - -import jakarta.servlet.http.HttpServletRequest; -import org.jetbrains.annotations.NotNull; -import org.jetbrains.annotations.Nullable; -import org.labkey.api.data.ContainerManager; -import org.labkey.api.module.DefaultModule; -import org.labkey.api.module.SupportedDatabase; -import org.labkey.api.util.ExceptionReportingLevel; -import org.labkey.api.util.Path; -import org.labkey.api.util.UsageReportingLevel; -import org.labkey.api.view.ActionURL; - -import java.io.File; -import java.util.List; -import java.util.Map; -import java.util.Set; - -/** - * Stores basic site-wide configuration. - * @see org.labkey.api.settings.WriteableAppProps - */ -public interface AppProps -{ - AppProps _instance = new AppPropsImpl(); - - String SCOPE_SITE_SETTINGS = "SiteSettings"; - - // Used for all optional features; "experimental" for historical reasons. - String OPTIONAL_FEATURE_PREFIX = "experimentalFeature."; - String SCOPE_OPTIONAL_FEATURE = "ExperimentalFeature"; // Startup property prefix for all optional features; "Experimental" for historical reasons. - String EXPERIMENTAL_NO_GUESTS = "disableGuestAccount"; - String EXPERIMENTAL_BLOCKER = "blockMaliciousClients"; - String EXPERIMENTAL_RESOLVE_PROPERTY_URI_COLUMNS = "resolve-property-uri-columns"; - String ADMIN_PROVIDED_ALLOWED_EXTERNAL_RESOURCES = "allowedExternalResources"; - String QUANTITY_COLUMN_SUFFIX_TESTING = "quantityColumnSuffixTesting"; - String GENERATE_CONTROLLER_FIRST_URLS = "generateControllerFirstUrls"; - String REJECT_CONTROLLER_FIRST_URLS = "rejectControllerFirstUrls"; - String MULTI_VALUE_TEXT_CHOICE = "multiChoiceDataType"; - - String UNKNOWN_VERSION = "Unknown Release Version"; - - static AppProps getInstance() - { - return _instance; - } - - static WriteableAppProps getWriteableInstance() - { - return new WriteableAppProps(ContainerManager.getRoot()); - } - - String getServerSessionGUID(); - - boolean isMailRecorderEnabled(); - - boolean isOptionalFeatureEnabled(String feature); - - boolean isDevMode(); - - @Nullable - String getEnlistmentId(); - - boolean isCachingAllowed(); - - boolean isRecompileJspEnabled(); - - /** - * Indicates whether modules' "sourcePath" and "buildPath" values be ignored. This allows a server to run in devMode - * without the risk of loading unwanted resources from a source tree that may not match the deployed server. - * - * WARNING: Setting this flag will interfere with the population of module beans, resulting in a - * mismatch between deployed modules and their properties on the server. - * - * @return value of the 'labkey.ignoreModuleSource' system property. Defaults to false - * - * @see org.labkey.api.module.DefaultModule#setSourcePath(String) - * @see org.labkey.api.module.DefaultModule#setBuildPath(String) - * @see DefaultModule#computeResourceDirectory() - */ - boolean isIgnoreModuleSource(); - - void setProjectRoot(String projectRoot); - - /** - * @return the root of the main source tree - */ - @Nullable - String getProjectRoot(); - - /** - * @return directory under which all containers will automatically have their own subdirectory for storing files - */ - @Nullable - File getFileSystemRoot(); - - @NotNull - UsageReportingLevel getUsageReportingLevel(); - - /** - * Returns the core module's release version, a string such as "20.3-SNAPSHOT", "20.1.0", or "20.3.7". - * Or "Unknown Release Version". - */ - @NotNull - String getReleaseVersion(); - - /** - * Convenience method for getting the core schema version, returning 0.0 instead of null - */ - double getSchemaVersion(); - - String getContextPath(); - - Path getParsedContextPath(); - - int getServerPort(); - - String getScheme(); - - String getServerName(); - - /** - * Save the current request URL if the base server URL property is not set - */ - void ensureBaseServerUrl(HttpServletRequest request); - - void setContextPath(String contextPath); - - boolean isSetBaseServerUrl(); - - String getBaseServerUrl(); - - String getHomePageUrl(); - - ActionURL getHomePageActionURL(); - - String getSiteWelcomePageUrlString(); - - int getLookAndFeelRevision(); - - String getDefaultLsidAuthority(); - - String getPipelineToolsDirectory(); - - boolean isSSLRequired(); - - boolean isUserRequestedAdminOnlyMode(); - - String getAdminOnlyMessage(); - - boolean isShowRibbonMessage(); - - @Nullable String getRibbonMessage(); - - int getSSLPort(); - - int getMemoryUsageDumpInterval(); - - /** Timeout in seconds for read-only HTTP requests, after which resources like DB connections and spawned processes will be killed. Set to 0 to disable. */ - int getReadOnlyHttpRequestTimeout(); - - int getMaxBLOBSize(); - - boolean isExt3Required(); - - boolean isExt3APIRequired(); - - ExceptionReportingLevel getExceptionReportingLevel(); - - /** - * Flag specifying if the project navigation access is open/closed. Open (default) means users will see the full - * folder tree for all folders they have permissions to see. Closed follows the rules as specified in issue #32718. - * - * @return if navigation access is open - */ - boolean isNavigationAccessOpen(); - - boolean isSelfReportExceptions(); - - String getServerGUID(); - - String getBLASTServerBaseURL(); - - /** @return the name of the Tomcat XML deployment descriptor based on the context path for this install - now always application.properties */ - String getWebappConfigurationFilename(); - - /** - * Email address of the primary site or application administrator, set on the site settings page. Useful in error - * messages when only an administrator can help. Returns null if there are no site or application admins (i.e., - * only impersonating troubleshooters). - * - * @return Email address of the primary site or application administrator - */ - @Nullable String getAdministratorContactEmail(boolean includeAppAdmins); - - /** - * Whether to use the newer, and preferred, container-relative style of URLs of the form - * /contextPath/container/controller-action.view, or the older controller-relative style like - * /contextPath/controller/container/action.view - */ - boolean getUseContainerRelativeURL(); - - boolean isAllowApiKeys(); - - int getApiKeyExpirationSeconds(); - - boolean isAllowSessionKeys(); - - // configurable http security settings - - /** - * @return "SAMEORIGIN" or "DENY" or "ALLOW" - */ - String getXFrameOption(); - - String getStaticFilesPrefix(); - - boolean isWebfilesRootEnabled(); - - boolean isFileUploadDisabled(); - - boolean isInvalidFilenameUploadBlocked(); - - boolean isInvalidFilenameBlocked(); - - /** @return whether the server should include its name and version as a header in HTTP responses */ - boolean isIncludeServerHttpHeader(); - - /** - * @return List of configured external redirect hosts - */ - @NotNull - List getExternalRedirectHosts(); - - /** - * @return List of configured external resource hosts - */ - @Deprecated // Left for upgrade code only - @NotNull - List getExternalSourceHosts(); - - Map getStashedStartupProperties(); - - @NotNull String getDistributionName(); - - @NotNull String getDistributionFilename(); - - @NotNull Set getDistributionSupportedDatabases(); - - @NotNull List getAllowedExtensions(); - - @NotNull String getAllowedExternalResourceHosts(); -} +/* + * Copyright (c) 2008-2019 LabKey Corporation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.labkey.api.settings; + +import jakarta.servlet.http.HttpServletRequest; +import org.jetbrains.annotations.NotNull; +import org.jetbrains.annotations.Nullable; +import org.labkey.api.data.ContainerManager; +import org.labkey.api.module.DefaultModule; +import org.labkey.api.module.SupportedDatabase; +import org.labkey.api.util.ExceptionReportingLevel; +import org.labkey.api.util.Path; +import org.labkey.api.util.UsageReportingLevel; +import org.labkey.api.view.ActionURL; + +import java.io.File; +import java.util.List; +import java.util.Map; +import java.util.Set; + +/** + * Stores basic site-wide configuration. + * @see org.labkey.api.settings.WriteableAppProps + */ +public interface AppProps +{ + AppProps _instance = new AppPropsImpl(); + + String SCOPE_SITE_SETTINGS = "SiteSettings"; + + // Used for all optional features; "experimental" for historical reasons. + String OPTIONAL_FEATURE_PREFIX = "experimentalFeature."; + String SCOPE_OPTIONAL_FEATURE = "ExperimentalFeature"; // Startup property prefix for all optional features; "Experimental" for historical reasons. + String EXPERIMENTAL_NO_GUESTS = "disableGuestAccount"; + String EXPERIMENTAL_BLOCKER = "blockMaliciousClients"; + String EXPERIMENTAL_RESOLVE_PROPERTY_URI_COLUMNS = "resolve-property-uri-columns"; + String ADMIN_PROVIDED_ALLOWED_EXTERNAL_RESOURCES = "allowedExternalResources"; + String QUANTITY_COLUMN_SUFFIX_TESTING = "quantityColumnSuffixTesting"; + String GENERATE_CONTROLLER_FIRST_URLS = "generateControllerFirstUrls"; + String REJECT_CONTROLLER_FIRST_URLS = "rejectControllerFirstUrls"; + String MULTI_VALUE_TEXT_CHOICE = "multiChoiceDataType"; + + String UNKNOWN_VERSION = "Unknown Release Version"; + + static AppProps getInstance() + { + return _instance; + } + + static WriteableAppProps getWriteableInstance() + { + return new WriteableAppProps(ContainerManager.getRoot()); + } + + String getServerSessionGUID(); + + boolean isMailRecorderEnabled(); + + boolean isOptionalFeatureEnabled(String feature); + + boolean isDevMode(); + + @Nullable + String getEnlistmentId(); + + boolean isCachingAllowed(); + + boolean isRecompileJspEnabled(); + + /** + * Indicates whether modules' "sourcePath" and "buildPath" values be ignored. This allows a server to run in devMode + * without the risk of loading unwanted resources from a source tree that may not match the deployed server. + * + * WARNING: Setting this flag will interfere with the population of module beans, resulting in a + * mismatch between deployed modules and their properties on the server. + * + * @return value of the 'labkey.ignoreModuleSource' system property. Defaults to false + * + * @see org.labkey.api.module.DefaultModule#setSourcePath(String) + * @see org.labkey.api.module.DefaultModule#setBuildPath(String) + * @see DefaultModule#computeResourceDirectory() + */ + boolean isIgnoreModuleSource(); + + void setProjectRoot(String projectRoot); + + /** + * @return the root of the main source tree + */ + @Nullable + String getProjectRoot(); + + /** + * @return directory under which all containers will automatically have their own subdirectory for storing files + */ + @Nullable + File getFileSystemRoot(); + + @NotNull + UsageReportingLevel getUsageReportingLevel(); + + /** + * Returns the core module's release version, a string such as "20.3-SNAPSHOT", "20.1.0", or "20.3.7". + * Or "Unknown Release Version". + */ + @NotNull + String getReleaseVersion(); + + /** + * Convenience method for getting the core schema version, returning 0.0 instead of null + */ + double getSchemaVersion(); + + String getContextPath(); + + Path getParsedContextPath(); + + int getServerPort(); + + String getScheme(); + + String getServerName(); + + /** + * Save the current request URL if the base server URL property is not set + */ + void ensureBaseServerUrl(HttpServletRequest request); + + void setContextPath(String contextPath); + + boolean isSetBaseServerUrl(); + + String getBaseServerUrl(); + + String getHomePageUrl(); + + ActionURL getHomePageActionURL(); + + String getSiteWelcomePageUrlString(); + + int getLookAndFeelRevision(); + + String getDefaultLsidAuthority(); + + String getPipelineToolsDirectory(); + + boolean isSSLRequired(); + + boolean isUserRequestedAdminOnlyMode(); + + String getAdminOnlyMessage(); + + boolean isShowRibbonMessage(); + + @Nullable String getRibbonMessage(); + + int getSSLPort(); + + int getMemoryUsageDumpInterval(); + + /** Timeout in seconds for read-only HTTP requests, after which resources like DB connections and spawned processes will be killed. Set to 0 to disable. */ + int getReadOnlyHttpRequestTimeout(); + + int getMaxBLOBSize(); + + boolean isExt3Required(); + + boolean isExt3APIRequired(); + + ExceptionReportingLevel getExceptionReportingLevel(); + + /** + * Flag specifying if the project navigation access is open/closed. Open (default) means users will see the full + * folder tree for all folders they have permissions to see. Closed follows the rules as specified in issue #32718. + * + * @return if navigation access is open + */ + boolean isNavigationAccessOpen(); + + boolean isSelfReportExceptions(); + + String getServerGUID(); + + String getBLASTServerBaseURL(); + + /** @return the name of the Tomcat XML deployment descriptor based on the context path for this install - now always application.properties */ + String getWebappConfigurationFilename(); + + /** + * Email address of the primary site or application administrator, set on the site settings page. Useful in error + * messages when only an administrator can help. Returns null if there are no site or application admins (i.e., + * only impersonating troubleshooters). + * + * @return Email address of the primary site or application administrator + */ + @Nullable String getAdministratorContactEmail(boolean includeAppAdmins); + + /** + * Whether to use the newer, and preferred, container-relative style of URLs of the form + * /contextPath/container/controller-action.view, or the older controller-relative style like + * /contextPath/controller/container/action.view + */ + boolean getUseContainerRelativeURL(); + + boolean isAllowApiKeys(); + + int getApiKeyExpirationSeconds(); + + boolean isAllowSessionKeys(); + + // configurable http security settings + + /** + * @return "SAMEORIGIN" or "DENY" or "ALLOW" + */ + String getXFrameOption(); + + String getStaticFilesPrefix(); + + boolean isWebfilesRootEnabled(); + + boolean isFileUploadDisabled(); + + boolean isInvalidFilenameUploadBlocked(); + + boolean isInvalidFilenameBlocked(); + + /** @return whether the server should include its name and version as a header in HTTP responses */ + boolean isIncludeServerHttpHeader(); + + /** + * @return List of configured external redirect hosts + */ + @NotNull + List getExternalRedirectHosts(); + + /** + * @return List of configured external resource hosts + */ + @Deprecated // Left for upgrade code only + @NotNull + List getExternalSourceHosts(); + + Map getStashedStartupProperties(); + + @NotNull String getDistributionName(); + + @NotNull String getDistributionFilename(); + + @NotNull Set getDistributionSupportedDatabases(); + + @NotNull List getAllowedExtensions(); + + @NotNull String getAllowedExternalResourceHosts(); +} diff --git a/experiment/src/org/labkey/experiment/ExperimentModule.java b/experiment/src/org/labkey/experiment/ExperimentModule.java index fb9cb2d52a4..4f70bc8b967 100644 --- a/experiment/src/org/labkey/experiment/ExperimentModule.java +++ b/experiment/src/org/labkey/experiment/ExperimentModule.java @@ -1,1131 +1,1131 @@ -/* - * Copyright (c) 2008-2019 LabKey Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.labkey.experiment; - -import org.apache.commons.lang3.math.NumberUtils; -import org.jetbrains.annotations.NotNull; -import org.jetbrains.annotations.Nullable; -import org.labkey.api.admin.FolderSerializationRegistry; -import org.labkey.api.assay.AssayProvider; -import org.labkey.api.assay.AssayService; -import org.labkey.api.attachments.AttachmentService; -import org.labkey.api.audit.AuditLogService; -import org.labkey.api.audit.SampleTimelineAuditEvent; -import org.labkey.api.collections.LongHashMap; -import org.labkey.api.data.Container; -import org.labkey.api.data.ContainerFilter; -import org.labkey.api.data.ContainerManager; -import org.labkey.api.data.CoreSchema; -import org.labkey.api.data.DbSchema; -import org.labkey.api.data.DbSchemaType; -import org.labkey.api.data.JdbcType; -import org.labkey.api.data.NameGenerator; -import org.labkey.api.data.SQLFragment; -import org.labkey.api.data.SimpleFilter; -import org.labkey.api.data.SimpleFilter.FilterClause; -import org.labkey.api.data.SqlSelector; -import org.labkey.api.data.TableInfo; -import org.labkey.api.data.TableSelector; -import org.labkey.api.data.UpgradeCode; -import org.labkey.api.defaults.DefaultValueService; -import org.labkey.api.exp.ExperimentException; -import org.labkey.api.exp.ExperimentRunType; -import org.labkey.api.exp.Lsid; -import org.labkey.api.exp.OntologyManager; -import org.labkey.api.exp.PropertyType; -import org.labkey.api.exp.api.DefaultExperimentDataHandler; -import org.labkey.api.exp.api.ExpData; -import org.labkey.api.exp.api.ExpDataClass; -import org.labkey.api.exp.api.ExpLineageService; -import org.labkey.api.exp.api.ExpMaterial; -import org.labkey.api.exp.api.ExpProtocol; -import org.labkey.api.exp.api.ExpProtocolAttachmentType; -import org.labkey.api.exp.api.ExpRunAttachmentType; -import org.labkey.api.exp.api.ExpSampleType; -import org.labkey.api.exp.api.ExperimentJSONConverter; -import org.labkey.api.exp.api.ExperimentService; -import org.labkey.api.exp.api.FilterProtocolInputCriteria; -import org.labkey.api.exp.api.SampleTypeDomainKind; -import org.labkey.api.exp.api.SampleTypeService; -import org.labkey.api.exp.api.StorageProvisioner; -import org.labkey.api.exp.property.DomainAuditProvider; -import org.labkey.api.exp.property.DomainPropertyAuditProvider; -import org.labkey.api.exp.property.ExperimentProperty; -import org.labkey.api.exp.property.PropertyService; -import org.labkey.api.exp.property.SystemProperty; -import org.labkey.api.exp.query.ExpDataClassTable; -import org.labkey.api.exp.query.ExpSampleTypeTable; -import org.labkey.api.exp.query.ExpSchema; -import org.labkey.api.exp.query.SamplesSchema; -import org.labkey.api.exp.xar.LSIDRelativizer; -import org.labkey.api.exp.xar.LsidUtils; -import org.labkey.api.files.FileContentService; -import org.labkey.api.files.TableUpdaterFileListener; -import org.labkey.api.migration.DatabaseMigrationService; -import org.labkey.api.migration.ExperimentDeleteService; -import org.labkey.api.migration.MigrationTableHandler; -import org.labkey.api.module.ModuleContext; -import org.labkey.api.module.ModuleLoader; -import org.labkey.api.module.SpringModule; -import org.labkey.api.module.Summary; -import org.labkey.api.ontology.OntologyService; -import org.labkey.api.ontology.Quantity; -import org.labkey.api.ontology.Unit; -import org.labkey.api.pipeline.PipelineService; -import org.labkey.api.query.FieldKey; -import org.labkey.api.query.FilteredTable; -import org.labkey.api.query.QueryService; -import org.labkey.api.query.UserSchema; -import org.labkey.api.search.SearchService; -import org.labkey.api.security.User; -import org.labkey.api.security.roles.RoleManager; -import org.labkey.api.settings.AppProps; -import org.labkey.api.settings.OptionalFeatureService; -import org.labkey.api.usageMetrics.UsageMetricsService; -import org.labkey.api.util.GUID; -import org.labkey.api.util.JspTestCase; -import org.labkey.api.util.PageFlowUtil; -import org.labkey.api.util.StringUtilsLabKey; -import org.labkey.api.util.SystemMaintenance; -import org.labkey.api.view.AlwaysAvailableWebPartFactory; -import org.labkey.api.view.BaseWebPartFactory; -import org.labkey.api.view.HttpView; -import org.labkey.api.view.JspView; -import org.labkey.api.view.Portal; -import org.labkey.api.view.ViewContext; -import org.labkey.api.view.WebPartFactory; -import org.labkey.api.view.WebPartView; -import org.labkey.api.view.template.WarningService; -import org.labkey.api.vocabulary.security.DesignVocabularyPermission; -import org.labkey.api.webdav.WebdavResource; -import org.labkey.api.webdav.WebdavService; -import org.labkey.experiment.api.DataClassDomainKind; -import org.labkey.experiment.api.ExpDataClassImpl; -import org.labkey.experiment.api.ExpDataClassTableImpl; -import org.labkey.experiment.api.ExpDataClassType; -import org.labkey.experiment.api.ExpDataImpl; -import org.labkey.experiment.api.ExpDataTableImpl; -import org.labkey.experiment.api.ExpMaterialImpl; -import org.labkey.experiment.api.ExpProtocolImpl; -import org.labkey.experiment.api.ExpSampleTypeImpl; -import org.labkey.experiment.api.ExpSampleTypeTableImpl; -import org.labkey.experiment.api.ExperimentServiceImpl; -import org.labkey.experiment.api.ExperimentStressTest; -import org.labkey.experiment.api.GraphAlgorithms; -import org.labkey.experiment.api.LineageTest; -import org.labkey.experiment.api.LogDataType; -import org.labkey.experiment.api.Protocol; -import org.labkey.experiment.api.SampleTypeServiceImpl; -import org.labkey.experiment.api.SampleTypeUpdateServiceDI; -import org.labkey.experiment.api.UniqueValueCounterTestCase; -import org.labkey.experiment.api.VocabularyDomainKind; -import org.labkey.experiment.api.data.ChildOfCompareType; -import org.labkey.experiment.api.data.ChildOfMethod; -import org.labkey.experiment.api.data.LineageCompareType; -import org.labkey.experiment.api.data.ParentOfCompareType; -import org.labkey.experiment.api.data.ParentOfMethod; -import org.labkey.experiment.api.property.DomainImpl; -import org.labkey.experiment.api.property.DomainPropertyImpl; -import org.labkey.experiment.api.property.LengthValidator; -import org.labkey.experiment.api.property.LookupValidator; -import org.labkey.experiment.api.property.PropertyServiceImpl; -import org.labkey.experiment.api.property.RangeValidator; -import org.labkey.experiment.api.property.RegExValidator; -import org.labkey.experiment.api.property.StorageNameGenerator; -import org.labkey.experiment.api.property.StorageProvisionerImpl; -import org.labkey.experiment.api.property.TextChoiceValidator; -import org.labkey.experiment.controllers.exp.ExperimentController; -import org.labkey.experiment.controllers.property.PropertyController; -import org.labkey.experiment.defaults.DefaultValueServiceImpl; -import org.labkey.experiment.lineage.ExpLineageServiceImpl; -import org.labkey.experiment.lineage.LineagePerfTest; -import org.labkey.experiment.pipeline.ExperimentPipelineProvider; -import org.labkey.experiment.pipeline.XarTestPipelineJob; -import org.labkey.experiment.samples.DataClassFolderImporter; -import org.labkey.experiment.samples.DataClassFolderWriter; -import org.labkey.experiment.samples.SampleStatusFolderImporter; -import org.labkey.experiment.samples.SampleTimelineAuditProvider; -import org.labkey.experiment.samples.SampleTypeFolderImporter; -import org.labkey.experiment.samples.SampleTypeFolderWriter; -import org.labkey.experiment.security.DataClassDesignerRole; -import org.labkey.experiment.security.SampleTypeDesignerRole; -import org.labkey.experiment.types.TypesController; -import org.labkey.experiment.xar.FolderXarImporterFactory; -import org.labkey.experiment.xar.FolderXarWriterFactory; - -import java.util.ArrayList; -import java.util.Collection; -import java.util.Collections; -import java.util.HashMap; -import java.util.HashSet; -import java.util.LinkedHashSet; -import java.util.LinkedList; -import java.util.List; -import java.util.Map; -import java.util.Set; -import java.util.function.Supplier; -import java.util.stream.Collectors; - -import static org.labkey.api.data.ColumnRenderPropertiesImpl.STORAGE_UNIQUE_ID_CONCEPT_URI; -import static org.labkey.api.data.ColumnRenderPropertiesImpl.TEXT_CHOICE_CONCEPT_URI; -import static org.labkey.api.exp.api.ExperimentService.MODULE_NAME; - -public class ExperimentModule extends SpringModule -{ - private static final String SAMPLE_TYPE_WEB_PART_NAME = "Sample Types"; - private static final String PROTOCOL_WEB_PART_NAME = "Protocols"; - - public static final String AMOUNT_AND_UNIT_UPGRADE_PROP = "AmountAndUnitAudit"; - public static final String TRANSACTION_ID_PROP = "AuditTransactionId"; - public static final String AUDIT_COUNT_PROP = "AuditRecordCount"; - public static final String EXPERIMENT_RUN_WEB_PART_NAME = "Experiment Runs"; - - @Override - public String getName() - { - return MODULE_NAME; - } - - @Override - public Double getSchemaVersion() - { - return 26.001; - } - - @Nullable - @Override - public UpgradeCode getUpgradeCode() - { - return new ExperimentUpgradeCode(); - } - - @Override - protected void init() - { - addController("experiment", ExperimentController.class); - addController("experiment-types", TypesController.class); - addController("property", PropertyController.class); - ExperimentService.setInstance(new ExperimentServiceImpl()); - SampleTypeService.setInstance(new SampleTypeServiceImpl()); - DefaultValueService.setInstance(new DefaultValueServiceImpl()); - StorageProvisioner.setInstance(StorageProvisionerImpl.get()); - ExpLineageService.setInstance(new ExpLineageServiceImpl()); - - PropertyServiceImpl propertyServiceImpl = new PropertyServiceImpl(); - PropertyService.setInstance(propertyServiceImpl); - UsageMetricsService.get().registerUsageMetrics(getName(), propertyServiceImpl); - - UsageMetricsService.get().registerUsageMetrics(getName(), FileLinkMetricsProvider.getInstance()); - - ExperimentProperty.register(); - SamplesSchema.register(this); - ExpSchema.register(this); - - PropertyService.get().registerDomainKind(new SampleTypeDomainKind()); - PropertyService.get().registerDomainKind(new DataClassDomainKind()); - PropertyService.get().registerDomainKind(new VocabularyDomainKind()); - - QueryService.get().addCompareType(new ChildOfCompareType()); - QueryService.get().addCompareType(new ParentOfCompareType()); - QueryService.get().addCompareType(new LineageCompareType()); - QueryService.get().registerMethod(ChildOfMethod.NAME, new ChildOfMethod(), JdbcType.BOOLEAN, 2, 3); - QueryService.get().registerMethod(ParentOfMethod.NAME, new ParentOfMethod(), JdbcType.BOOLEAN, 2, 3); - QueryService.get().addQueryListener(new ExperimentQueryChangeListener()); - QueryService.get().addQueryListener(new PropertyQueryChangeListener()); - - PropertyService.get().registerValidatorKind(new RegExValidator()); - PropertyService.get().registerValidatorKind(new RangeValidator()); - PropertyService.get().registerValidatorKind(new LookupValidator()); - PropertyService.get().registerValidatorKind(new LengthValidator()); - PropertyService.get().registerValidatorKind(new TextChoiceValidator()); - - ExperimentService.get().registerExperimentDataHandler(new DefaultExperimentDataHandler()); - ExperimentService.get().registerProtocolInputCriteria(new FilterProtocolInputCriteria.Factory()); - ExperimentService.get().registerNameExpressionType("sampletype", "exp", "MaterialSource", "nameexpression"); - ExperimentService.get().registerNameExpressionType("aliquots", "exp", "MaterialSource", "aliquotnameexpression"); - ExperimentService.get().registerNameExpressionType("dataclass", "exp", "DataClass", "nameexpression"); - - OptionalFeatureService.get().addExperimentalFeatureFlag(AppProps.EXPERIMENTAL_RESOLVE_PROPERTY_URI_COLUMNS, "Resolve property URIs as columns on experiment tables", - "If a column is not found on an experiment table, attempt to resolve the column name as a Property URI and add it as a property column", false); - if (CoreSchema.getInstance().getSqlDialect().isSqlServer()) - { - OptionalFeatureService.get().addExperimentalFeatureFlag(NameGenerator.EXPERIMENTAL_WITH_COUNTER, "Use strict incremental withCounter and rootSampleCount expression", - "When withCounter or rootSampleCount is used in name expression, make sure the count increments one-by-one and does not jump.", true); - } - else - { - OptionalFeatureService.get().addExperimentalFeatureFlag(NameGenerator.EXPERIMENTAL_ALLOW_GAP_COUNTER, "Allow gap with withCounter and rootSampleCount expression", - "Check this option if gaps in the count generated by withCounter or rootSampleCount name expression are allowed.", true); - OptionalFeatureService.get().addExperimentalFeatureFlag(AppProps.MULTI_VALUE_TEXT_CHOICE, "Allow multi-value Text Choice properties", - "Support selecting more than one values for text choice fields", false); - } - OptionalFeatureService.get().addExperimentalFeatureFlag(AppProps.QUANTITY_COLUMN_SUFFIX_TESTING, "Quantity column suffix testing", - "If a column name contains a \"__\" suffix, this feature allows for testing it as a Quantity display column", false); - OptionalFeatureService.get().addExperimentalFeatureFlag(ExperimentService.EXPERIMENTAL_FEATURE_FROM_EXPANCESTORS, "SQL syntax: 'FROM EXPANCESTORS()'", - "Support for querying lineage of experiment objects", false); - OptionalFeatureService.get().addExperimentalFeatureFlag(SampleTypeUpdateServiceDI.EXPERIMENTAL_FEATURE_ALLOW_ROW_ID_SAMPLE_MERGE, "Allow RowId to be accepted when merging samples", - "If the incoming data includes a RowId column we will allow the column but ignore it's values.", false); - - RoleManager.registerPermission(new DesignVocabularyPermission(), true); - RoleManager.registerRole(new SampleTypeDesignerRole()); - RoleManager.registerRole(new DataClassDesignerRole()); - - AttachmentService.get().registerAttachmentParentType(ExpRunAttachmentType.get()); - AttachmentService.get().registerAttachmentParentType(ExpProtocolAttachmentType.get()); - - WebdavService.get().addExpDataProvider((path, container) -> ExperimentService.get().getAllExpDataByURL(path, container)); - ExperimentService.get().registerObjectReferencer(ExperimentServiceImpl.get()); - - addModuleProperty(new LineageMaximumDepthModuleProperty(this)); - WarningService.get().register(new ExperimentWarningProvider()); - } - - @Override - public boolean hasScripts() - { - return true; - } - - @Override - @NotNull - protected Collection createWebPartFactories() - { - List result = new ArrayList<>(); - - BaseWebPartFactory runGroupsFactory = new BaseWebPartFactory(RunGroupWebPart.WEB_PART_NAME, WebPartFactory.LOCATION_BODY, WebPartFactory.LOCATION_RIGHT) - { - @Override - public WebPartView getWebPartView(@NotNull ViewContext portalCtx, @NotNull Portal.WebPart webPart) - { - return new RunGroupWebPart(portalCtx, WebPartFactory.LOCATION_RIGHT.equalsIgnoreCase(webPart.getLocation()), webPart); - } - }; - runGroupsFactory.addLegacyNames("Experiments", "Experiment", "Experiment Navigator", "Narrow Experiments"); - result.add(runGroupsFactory); - - BaseWebPartFactory runTypesFactory = new BaseWebPartFactory(RunTypeWebPart.WEB_PART_NAME, WebPartFactory.LOCATION_BODY, WebPartFactory.LOCATION_RIGHT) - { - @Override - public WebPartView getWebPartView(@NotNull ViewContext portalCtx, @NotNull Portal.WebPart webPart) - { - return new RunTypeWebPart(); - } - }; - result.add(runTypesFactory); - - result.add(new ExperimentRunWebPartFactory()); - BaseWebPartFactory sampleTypeFactory = new BaseWebPartFactory(SAMPLE_TYPE_WEB_PART_NAME, WebPartFactory.LOCATION_BODY, WebPartFactory.LOCATION_RIGHT) - { - @Override - public WebPartView getWebPartView(@NotNull ViewContext portalCtx, @NotNull Portal.WebPart webPart) - { - return new SampleTypeWebPart(WebPartFactory.LOCATION_RIGHT.equalsIgnoreCase(webPart.getLocation()), portalCtx); - } - }; - sampleTypeFactory.addLegacyNames("Narrow Sample Sets", "Sample Sets"); - result.add(sampleTypeFactory); - result.add(new AlwaysAvailableWebPartFactory("Samples Menu", false, false, WebPartFactory.LOCATION_MENUBAR) { - @Override - public WebPartView getWebPartView(@NotNull ViewContext portalCtx, @NotNull Portal.WebPart webPart) - { - WebPartView view = new JspView<>("/org/labkey/experiment/samplesAndAnalytes.jsp", webPart); - view.setTitle("Samples"); - return view; - } - }); - - result.add(new AlwaysAvailableWebPartFactory("Data Classes", false, false, WebPartFactory.LOCATION_BODY, WebPartFactory.LOCATION_RIGHT) { - @Override - public WebPartView getWebPartView(@NotNull ViewContext portalCtx, @NotNull Portal.WebPart webPart) - { - return new DataClassWebPart(WebPartFactory.LOCATION_RIGHT.equalsIgnoreCase(webPart.getLocation()), portalCtx, webPart); - } - }); - - BaseWebPartFactory narrowProtocolFactory = new BaseWebPartFactory(PROTOCOL_WEB_PART_NAME, WebPartFactory.LOCATION_RIGHT) - { - @Override - public WebPartView getWebPartView(@NotNull ViewContext portalCtx, @NotNull Portal.WebPart webPart) - { - return new ProtocolWebPart(WebPartFactory.LOCATION_RIGHT.equalsIgnoreCase(webPart.getLocation()), portalCtx); - } - }; - narrowProtocolFactory.addLegacyNames("Narrow Protocols"); - result.add(narrowProtocolFactory); - - return result; - } - - private void addDataResourceResolver(String categoryName) - { - SearchService.get().addResourceResolver(categoryName, new SearchService.ResourceResolver() - { - @Override - public WebdavResource resolve(@NotNull String resourceIdentifier) - { - ExpDataImpl data = ExpDataImpl.fromDocumentId(resourceIdentifier); - if (data == null) - return null; - - return data.createIndexDocument(null); - } - - @Override - public Map getCustomSearchJson(User user, @NotNull String resourceIdentifier) - { - ExpDataImpl data = ExpDataImpl.fromDocumentId(resourceIdentifier); - if (data == null) - return null; - - return ExperimentJSONConverter.serializeData(data, user, ExperimentJSONConverter.DEFAULT_SETTINGS).toMap(); - } - - @Override - public Map> getCustomSearchJsonMap(User user, @NotNull Collection resourceIdentifiers) - { - Map idDataMap = ExpDataImpl.fromDocumentIds(resourceIdentifiers); - if (idDataMap == null) - return null; - - Map> searchJsonMap = new HashMap<>(); - for (String resourceIdentifier : idDataMap.keySet()) - searchJsonMap.put(resourceIdentifier, ExperimentJSONConverter.serializeData(idDataMap.get(resourceIdentifier), user, ExperimentJSONConverter.DEFAULT_SETTINGS).toMap()); - return searchJsonMap; - } - }); - } - - private void addDataClassResourceResolver(String categoryName) - { - SearchService.get().addResourceResolver(categoryName, new SearchService.ResourceResolver(){ - @Override - public Map getCustomSearchJson(User user, @NotNull String resourceIdentifier) - { - int rowId = NumberUtils.toInt(resourceIdentifier.replace(categoryName + ":", "")); - if (rowId == 0) - return null; - - ExpDataClass dataClass = ExperimentService.get().getDataClass(rowId); - if (dataClass == null) - return null; - - Map properties = ExperimentJSONConverter.serializeExpObject(dataClass, null, ExperimentJSONConverter.DEFAULT_SETTINGS, user).toMap(); - - //Need to map to proper Icon - properties.put("type", "dataClass" + (dataClass.getCategory() != null ? ":" + dataClass.getCategory() : "")); - - return properties; - } - }); - } - - private void addSampleTypeResourceResolver(String categoryName) - { - SearchService.get().addResourceResolver(categoryName, new SearchService.ResourceResolver(){ - @Override - public Map getCustomSearchJson(User user, @NotNull String resourceIdentifier) - { - int rowId = NumberUtils.toInt(resourceIdentifier.replace(categoryName + ":", "")); - if (rowId == 0) - return null; - - ExpSampleType sampleType = SampleTypeService.get().getSampleType(rowId); - if (sampleType == null) - return null; - - Map properties = ExperimentJSONConverter.serializeExpObject(sampleType, null, ExperimentJSONConverter.DEFAULT_SETTINGS, user).toMap(); - - //Need to map to proper Icon - properties.put("type", "sampleSet"); - - return properties; - } - }); - } - - private void addSampleResourceResolver(String categoryName) - { - SearchService.get().addResourceResolver(categoryName, new SearchService.ResourceResolver(){ - @Override - public Map getCustomSearchJson(User user, @NotNull String resourceIdentifier) - { - int rowId = NumberUtils.toInt(resourceIdentifier.replace(categoryName + ":", "")); - if (rowId == 0) - return null; - - ExpMaterial material = ExperimentService.get().getExpMaterial(rowId); - if (material == null) - return null; - - return ExperimentJSONConverter.serializeMaterial(material, user, ExperimentJSONConverter.DEFAULT_SETTINGS).toMap(); - } - - @Override - public Map> getCustomSearchJsonMap(User user, @NotNull Collection resourceIdentifiers) - { - Set rowIds = new HashSet<>(); - Map rowIdIdentifierMap = new LongHashMap<>(); - for (String resourceIdentifier : resourceIdentifiers) - { - long rowId = NumberUtils.toLong(resourceIdentifier.replace(categoryName + ":", "")); - if (rowId != 0) - { - rowIds.add(rowId); - rowIdIdentifierMap.put(rowId, resourceIdentifier); - } - } - - Map> searchJsonMap = new HashMap<>(); - for (ExpMaterial material : ExperimentService.get().getExpMaterials(rowIds)) - { - searchJsonMap.put( - rowIdIdentifierMap.get(material.getRowId()), - ExperimentJSONConverter.serializeMaterial(material, user, ExperimentJSONConverter.DEFAULT_SETTINGS).toMap() - ); - } - - return searchJsonMap; - } - }); - } - - @Override - protected void startupAfterSpringConfig(ModuleContext moduleContext) - { - SearchService ss = SearchService.get(); -// ss.addSearchCategory(OntologyManager.conceptCategory); - ss.addSearchCategory(ExpSampleTypeImpl.searchCategory); - ss.addSearchCategory(ExpSampleTypeImpl.mediaSearchCategory); - ss.addSearchCategory(ExpMaterialImpl.searchCategory); - ss.addSearchCategory(ExpMaterialImpl.mediaSearchCategory); - ss.addSearchCategory(ExpDataClassImpl.SEARCH_CATEGORY); - ss.addSearchCategory(ExpDataClassImpl.MEDIA_SEARCH_CATEGORY); - ss.addSearchCategory(ExpDataImpl.expDataCategory); - ss.addSearchCategory(ExpDataImpl.expMediaDataCategory); - ss.addSearchResultTemplate(new ExpDataImpl.DataSearchResultTemplate()); - addDataResourceResolver(ExpDataImpl.expDataCategory.getName()); - addDataResourceResolver(ExpDataImpl.expMediaDataCategory.getName()); - addDataClassResourceResolver(ExpDataClassImpl.SEARCH_CATEGORY.getName()); - addDataClassResourceResolver(ExpDataClassImpl.MEDIA_SEARCH_CATEGORY.getName()); - addSampleTypeResourceResolver(ExpSampleTypeImpl.searchCategory.getName()); - addSampleTypeResourceResolver(ExpSampleTypeImpl.mediaSearchCategory.getName()); - addSampleResourceResolver(ExpMaterialImpl.searchCategory.getName()); - addSampleResourceResolver(ExpMaterialImpl.mediaSearchCategory.getName()); - ss.addDocumentProvider(ExperimentServiceImpl.get()); - - PipelineService.get().registerPipelineProvider(new ExperimentPipelineProvider(this)); - ExperimentService.get().registerExperimentRunTypeSource(container -> Collections.singleton(ExperimentRunType.ALL_RUNS_TYPE)); - ExperimentService.get().registerDataType(new LogDataType()); - - AuditLogService.get().registerAuditType(new DomainAuditProvider()); - AuditLogService.get().registerAuditType(new DomainPropertyAuditProvider()); - AuditLogService.get().registerAuditType(new ExperimentAuditProvider()); - AuditLogService.get().registerAuditType(new SampleTypeAuditProvider()); - AuditLogService.get().registerAuditType(new SampleTimelineAuditProvider()); - - FileContentService fileContentService = FileContentService.get(); - if (null != fileContentService) - { - fileContentService.addFileListener(new ExpDataFileListener()); - fileContentService.addFileListener(new TableUpdaterFileListener(ExperimentService.get().getTinfoExperimentRun(), "FilePathRoot", TableUpdaterFileListener.Type.fileRootPath, "RowId")); - fileContentService.addFileListener(new FileLinkFileListener()); - } - ContainerManager.addContainerListener(new ContainerManager.ContainerListener() - { - @Override - public void containerDeleted(Container c, User user) - { - try - { - ExperimentService.get().deleteAllExpObjInContainer(c, user); - } - catch (ExperimentException ee) - { - throw new RuntimeException(ee); - } - } - }, - // This is in the Last group because when a container is deleted, - // the Experiment listener needs to be called after the Study listener, - // because Study needs the metadata held by Experiment to delete properly. - // but it should be before the CoreContainerListener - ContainerManager.ContainerListener.Order.Last); - - if (ModuleLoader.getInstance().shouldInsertData()) - SystemProperty.registerProperties(); - - FolderSerializationRegistry folderRegistry = FolderSerializationRegistry.get(); - if (null != folderRegistry) - { - folderRegistry.addFactories(new FolderXarWriterFactory(), new FolderXarImporterFactory()); - folderRegistry.addWriterFactory(new SampleTypeFolderWriter.SampleTypeDesignWriter.Factory()); - folderRegistry.addWriterFactory(new SampleTypeFolderWriter.SampleTypeDataWriter.Factory()); - folderRegistry.addWriterFactory(new DataClassFolderWriter.DataClassDesignWriter.Factory()); - folderRegistry.addWriterFactory(new DataClassFolderWriter.DataClassDataWriter.Factory()); - folderRegistry.addImportFactory(new SampleTypeFolderImporter.Factory()); - folderRegistry.addImportFactory(new DataClassFolderImporter.Factory()); - folderRegistry.addImportFactory(new SampleStatusFolderImporter.Factory()); - } - - AttachmentService.get().registerAttachmentParentType(ExpDataClassType.get()); - - WebdavService.get().addProvider(new ScriptsResourceProvider()); - - SystemMaintenance.addTask(new FileLinkMetricsMaintenanceTask()); - - UsageMetricsService svc = UsageMetricsService.get(); - if (null != svc) - { - svc.registerUsageMetrics(getName(), () -> { - Map results = new HashMap<>(); - - DbSchema schema = ExperimentService.get().getSchema(); - if (AssayService.get() != null) - { - Map assayMetrics = new HashMap<>(); - SQLFragment baseRunSQL = new SQLFragment("SELECT COUNT(*) FROM ").append(ExperimentService.get().getTinfoExperimentRun(), "r").append(" WHERE lsid LIKE ?"); - SQLFragment baseProtocolSQL = new SQLFragment("SELECT * FROM ").append(ExperimentService.get().getTinfoProtocol(), "p").append(" WHERE lsid LIKE ? AND ApplicationType = ?"); - for (AssayProvider assayProvider : AssayService.get().getAssayProviders()) - { - Map protocolMetrics = new HashMap<>(); - - // Run count across all assay designs of this type - SQLFragment runSQL = new SQLFragment(baseRunSQL); - runSQL.add(Lsid.namespaceLikeString(assayProvider.getRunLSIDPrefix())); - protocolMetrics.put("runCount", new SqlSelector(schema, runSQL).getObject(Long.class)); - - // Number of assay designs of this type - SQLFragment protocolSQL = new SQLFragment(baseProtocolSQL); - protocolSQL.add(assayProvider.getProtocolPattern()); - protocolSQL.add(ExpProtocol.ApplicationType.ExperimentRun.toString()); - List protocols = new SqlSelector(schema, protocolSQL).getArrayList(Protocol.class); - protocolMetrics.put("protocolCount", protocols.size()); - - List wrappedProtocols = protocols.stream().map(ExpProtocolImpl::new).collect(Collectors.toList()); - - protocolMetrics.put("resultRowCount", assayProvider.getResultRowCount(wrappedProtocols)); - - // Primary implementation class - protocolMetrics.put("implementingClass", assayProvider.getClass()); - - assayMetrics.put(assayProvider.getName(), protocolMetrics); - } - assayMetrics.put("autoLinkedAssayCount", new SqlSelector(schema, "SELECT COUNT(*) FROM exp.protocol EP JOIN exp.objectPropertiesView OP ON EP.lsid = OP.objecturi WHERE OP.propertyuri = 'terms.labkey.org#AutoCopyTargetContainer'").getObject(Long.class)); - assayMetrics.put("protocolsWithTransformScriptCount", new SqlSelector(schema, "SELECT COUNT(*) FROM exp.protocol EP JOIN exp.objectPropertiesView OP ON EP.lsid = OP.objecturi WHERE OP.name = 'TransformScript' AND status = 'Active'").getObject(Long.class)); - assayMetrics.put("protocolsWithTransformScriptRunOnEditCount", new SqlSelector(schema, "SELECT COUNT(*) FROM exp.protocol EP JOIN exp.objectPropertiesView OP ON EP.lsid = OP.objecturi WHERE OP.name = 'TransformScript' AND status = 'Active' AND OP.stringvalue LIKE '%\"INSERT\"%'").getObject(Long.class)); - assayMetrics.put("protocolsWithTransformScriptRunOnImportCount", new SqlSelector(schema, "SELECT COUNT(*) FROM exp.protocol EP JOIN exp.objectPropertiesView OP ON EP.lsid = OP.objecturi WHERE OP.name = 'TransformScript' AND status = 'Active' AND OP.stringvalue LIKE '%\"INSERT\"%'").getObject(Long.class)); - - assayMetrics.put("standardAssayWithPlateSupportCount", new SqlSelector(schema, "SELECT COUNT(*) FROM exp.protocol EP JOIN exp.objectPropertiesView OP ON EP.lsid = OP.objecturi WHERE OP.name = 'PlateMetadata' AND floatValue = 1").getObject(Long.class)); - SQLFragment runsWithPlateSQL = new SQLFragment(""" - SELECT COUNT(*) FROM exp.experimentrun r - INNER JOIN exp.object o ON o.objectUri = r.lsid - INNER JOIN exp.objectproperty op ON op.objectId = o.objectId - WHERE op.propertyid IN ( - SELECT propertyid FROM exp.propertydescriptor WHERE name = ? AND lookupquery = ? - )"""); - assayMetrics.put("standardAssayRunsWithPlateTemplate", new SqlSelector(schema, new SQLFragment(runsWithPlateSQL).add("PlateTemplate").add("PlateTemplate")).getObject(Long.class)); - assayMetrics.put("standardAssayRunsWithPlateSet", new SqlSelector(schema, new SQLFragment(runsWithPlateSQL).add("PlateSet").add("PlateSet")).getObject(Long.class)); - - assayMetrics.put("assayRunsFileColumnCount", new SqlSelector(schema, """ - SELECT COUNT(DISTINCT DD.DomainURI) FROM - exp.PropertyDescriptor D\s - JOIN exp.PropertyDomain PD ON D.propertyId = PD.propertyid - JOIN exp.DomainDescriptor DD on PD.domainID = DD.domainId - WHERE DD.domainUri LIKE ? AND D.rangeURI = ?""", "urn:lsid:%:" + ExpProtocol.AssayDomainTypes.Run.getPrefix() + ".%", PropertyType.FILE_LINK.getTypeUri()).getObject(Long.class)); - - assayMetrics.put("assayResultsFileColumnCount", new SqlSelector(schema, """ - SELECT COUNT(DISTINCT DD.DomainURI) FROM - exp.PropertyDescriptor D\s - JOIN exp.PropertyDomain PD ON D.propertyId = PD.propertyid - JOIN exp.DomainDescriptor DD on PD.domainID = DD.domainId - WHERE DD.domainUri LIKE ? AND D.rangeURI = ?""", "urn:lsid:%:" + ExpProtocol.AssayDomainTypes.Result.getPrefix() + ".%", PropertyType.FILE_LINK.getTypeUri()).getObject(Long.class)); - - Map sampleLookupCountMetrics = new HashMap<>(); - SQLFragment baseAssaySampleLookupSQL = new SQLFragment("SELECT COUNT(*) FROM exp.propertydescriptor WHERE (lookupschema = 'samples' OR (lookupschema = 'exp' AND lookupquery = 'Materials')) AND propertyuri LIKE ?"); - - SQLFragment batchAssaySampleLookupSQL = new SQLFragment(baseAssaySampleLookupSQL); - batchAssaySampleLookupSQL.add("urn:lsid:%:" + ExpProtocol.AssayDomainTypes.Batch.getPrefix() + ".%"); - sampleLookupCountMetrics.put("batchDomain", new SqlSelector(schema, batchAssaySampleLookupSQL).getObject(Long.class)); - - SQLFragment runAssaySampleLookupSQL = new SQLFragment(baseAssaySampleLookupSQL); - runAssaySampleLookupSQL.add("urn:lsid:%:" + ExpProtocol.AssayDomainTypes.Run.getPrefix() + ".%"); - sampleLookupCountMetrics.put("runDomain", new SqlSelector(schema, runAssaySampleLookupSQL).getObject(Long.class)); - - SQLFragment resultAssaySampleLookupSQL = new SQLFragment(baseAssaySampleLookupSQL); - resultAssaySampleLookupSQL.add("urn:lsid:%:" + ExpProtocol.AssayDomainTypes.Result.getPrefix() + ".%"); - sampleLookupCountMetrics.put("resultDomain", new SqlSelector(schema, resultAssaySampleLookupSQL).getObject(Long.class)); - - SQLFragment resultAssayMultipleSampleLookupSQL = new SQLFragment( - """ - SELECT COUNT(*) FROM ( - SELECT PD.domainid, COUNT(*) AS PropCount - FROM exp.propertydescriptor D - JOIN exp.PropertyDomain PD ON D.propertyId = PD.propertyid - WHERE (lookupschema = 'samples' OR (lookupschema = 'exp' AND lookupquery = 'Materials')) - AND propertyuri LIKE ? - GROUP BY PD.domainid - ) X WHERE X.PropCount > 1""" - ); - resultAssayMultipleSampleLookupSQL.add("urn:lsid:%:" + ExpProtocol.AssayDomainTypes.Result.getPrefix() + ".%"); - sampleLookupCountMetrics.put("resultDomainWithMultiple", new SqlSelector(schema, resultAssayMultipleSampleLookupSQL).getObject(Long.class)); - - assayMetrics.put("sampleLookupCount", sampleLookupCountMetrics); - - - // Putting these metrics at the same level as the other BooleanColumnCount metrics (e.g., sampleTypeWithBooleanColumnCount) - results.put("assayResultWithBooleanColumnCount", new SqlSelector(schema, """ - SELECT COUNT(DISTINCT DD.DomainURI) FROM - exp.PropertyDescriptor D\s - JOIN exp.PropertyDomain PD ON D.propertyId = PD.propertyid - JOIN exp.DomainDescriptor DD on PD.domainID = DD.domainId - WHERE D.propertyURI LIKE ? AND D.rangeURI = ?""", "urn:lsid:%:" + ExpProtocol.AssayDomainTypes.Result.getPrefix() + ".%", PropertyType.BOOLEAN.getTypeUri()).getObject(Long.class)); - - results.put("assayRunWithBooleanColumnCount", new SqlSelector(schema, """ - SELECT COUNT(DISTINCT DD.DomainURI) FROM - exp.PropertyDescriptor D\s - JOIN exp.PropertyDomain PD ON D.propertyId = PD.propertyid - JOIN exp.DomainDescriptor DD on PD.domainID = DD.domainId - WHERE D.propertyURI LIKE ? AND D.rangeURI = ?""", "urn:lsid:%:" + ExpProtocol.AssayDomainTypes.Run.getPrefix() + ".%", PropertyType.BOOLEAN.getTypeUri()).getObject(Long.class)); - - results.put("assay", assayMetrics); - } - - results.put("autoLinkedSampleSetCount", new SqlSelector(schema, "SELECT COUNT(*) FROM exp.materialsource WHERE autoLinkTargetContainer IS NOT NULL").getObject(Long.class)); - results.put("sampleSetCount", new SqlSelector(schema, "SELECT COUNT(*) FROM exp.materialsource").getObject(Long.class)); - - if (schema.getSqlDialect().isPostgreSQL()) // SQLServer does not support regular expression queries - { - Collection> numSampleCounts = new SqlSelector(schema, """ - SELECT totalCount, numberNameCount FROM - (SELECT cpastype, COUNT(*) AS totalCount from exp.material GROUP BY cpastype) t - JOIN - (SELECT cpastype, COUNT(*) AS numberNameCount FROM exp.material m WHERE m.name SIMILAR TO '[0-9.]*' GROUP BY cpastype) ns - ON t.cpastype = ns.cpastype""").getMapCollection(); - results.put("sampleSetWithNumberNamesCount", numSampleCounts.size()); - results.put("sampleSetWithOnlyNumberNamesCount", numSampleCounts.stream().filter( - map -> (Long) map.get("totalCount") > 0 && map.get("totalCount") == map.get("numberNameCount") - ).count()); - } - UserSchema userSchema = AuditLogService.getAuditLogSchema(User.getSearchUser(), ContainerManager.getRoot()); - FilteredTable table = (FilteredTable) userSchema.getTable(SampleTimelineAuditEvent.EVENT_TYPE); - - SQLFragment sql = new SQLFragment("SELECT COUNT(*)\n" + - " FROM (\n" + - " -- updates that are marked as lineage updates\n" + - " (SELECT DISTINCT transactionId\n" + - " FROM " + table.getRealTable().getFromSQL("").getSQL() +"\n" + - " WHERE islineageupdate = " + schema.getSqlDialect().getBooleanTRUE() + "\n" + - " AND comment = 'Sample was updated.'\n" + - " ) a1\n" + - " JOIN\n" + - " -- but have associated entries that are not lineage updates\n" + - " (SELECT DISTINCT transactionid\n" + - " FROM " + table.getRealTable().getFromSQL("").getSQL() + "\n" + - " WHERE islineageupdate = " + schema.getSqlDialect().getBooleanFALSE() + ") a2\n" + - " ON a1.transactionid = a2.transactionid\n" + - " )"); - - results.put("sampleLineageAuditDiscrepancyCount", new SqlSelector(schema, sql.getSQL()).getObject(Long.class)); - - results.put("sampleCount", new SqlSelector(schema, "SELECT COUNT(*) FROM exp.material").getObject(Long.class)); - results.put("aliquotCount", new SqlSelector(schema, "SELECT COUNT(*) FROM exp.material where aliquotedfromlsid IS NOT NULL").getObject(Long.class)); - results.put("sampleNullAmountCount", new SqlSelector(schema, "SELECT COUNT(*) FROM exp.material WHERE storedamount IS NULL").getObject(Long.class)); - results.put("sampleNegativeAmountCount", new SqlSelector(schema, "SELECT COUNT(*) FROM exp.material WHERE storedamount < 0").getObject(Long.class)); - results.put("sampleUnitsDifferCount", new SqlSelector(schema, "SELECT COUNT(*) from exp.material m JOIN exp.materialSource s ON m.materialsourceid = s.rowid WHERE m.units != s.metricunit").getObject(Long.class)); - results.put("sampleTypesWithoutUnitsCount", new SqlSelector(schema, "SELECT COUNT(*) from exp.materialSource WHERE category IS NULL AND metricunit IS NULL").getObject(Long.class)); - results.put("sampleTypesWithMassTypeUnit", new SqlSelector(schema, "SELECT COUNT(*) from exp.materialSource WHERE category IS NULL AND metricunit IN ('kg', 'g', 'mg', 'ug', 'ng')").getObject(Long.class)); - results.put("sampleTypesWithVolumeTypeUnit", new SqlSelector(schema, "SELECT COUNT(*) from exp.materialSource WHERE category IS NULL AND metricunit IN ('L', 'mL', 'uL')").getObject(Long.class)); - results.put("sampleTypesWithCountTypeUnit", new SqlSelector(schema, "SELECT COUNT(*) from exp.materialSource WHERE category IS NULL AND metricunit = ?", "unit").getObject(Long.class)); - - results.put("duplicateSampleMaterialNameCount", new SqlSelector(schema, "SELECT COUNT(*) as duplicateCount FROM " + - "(SELECT name, cpastype FROM exp.material WHERE cpastype <> 'Material' GROUP BY name, cpastype HAVING COUNT(*) > 1) d").getObject(Long.class)); - results.put("duplicateSpecimenMaterialNameCount", new SqlSelector(schema, "SELECT COUNT(*) as duplicateCount FROM " + - "(SELECT name, cpastype FROM exp.material WHERE cpastype = 'Material' GROUP BY name, cpastype HAVING COUNT(*) > 1) d").getObject(Long.class)); - String duplicateCaseInsensitiveSampleNameCountSql = """ - SELECT COUNT(*) FROM - ( - SELECT 1 AS found - FROM exp.material - WHERE materialsourceid IS NOT NULL - GROUP BY LOWER(name), materialsourceid - HAVING COUNT(*) > 1 - ) AS duplicates - """; - String duplicateCaseInsensitiveDataNameCountSql = """ - SELECT COUNT(*) FROM - ( - SELECT 1 AS found - FROM exp.data - WHERE classid IS NOT NULL - GROUP BY LOWER(name), classid - HAVING COUNT(*) > 1 - ) AS duplicates - """; - results.put("duplicateCaseInsensitiveSampleNameCount", new SqlSelector(schema, duplicateCaseInsensitiveSampleNameCountSql).getObject(Long.class)); - results.put("duplicateCaseInsensitiveDataNameCount", new SqlSelector(schema, duplicateCaseInsensitiveDataNameCountSql).getObject(Long.class)); - - results.put("dataClassCount", new SqlSelector(schema, "SELECT COUNT(*) FROM exp.dataclass").getObject(Long.class)); - results.put("dataClassRowCount", new SqlSelector(schema, "SELECT COUNT(*) FROM exp.data WHERE classid IN (SELECT rowid FROM exp.dataclass)").getObject(Long.class)); - results.put("dataWithDataParentsCount", new SqlSelector(schema, "SELECT COUNT(DISTINCT d.sourceApplicationId) FROM exp.data d\n" + - "JOIN exp.datainput di ON di.targetapplicationid = d.sourceapplicationid").getObject(Long.class)); - if (schema.getSqlDialect().isPostgreSQL()) - { - Collection> numDataClassObjectsCounts = new SqlSelector(schema, """ - SELECT totalCount, numberNameCount FROM - (SELECT cpastype, COUNT(*) AS totalCount from exp.data GROUP BY cpastype) t - JOIN - (SELECT cpastype, COUNT(*) AS numberNameCount FROM exp.data m WHERE m.name SIMILAR TO '[0-9.]*' GROUP BY cpastype) ns - ON t.cpastype = ns.cpastype""").getMapCollection(); - results.put("dataClassWithNumberNamesCount", numDataClassObjectsCounts.size()); - results.put("dataClassWithOnlyNumberNamesCount", numDataClassObjectsCounts.stream().filter(map -> - (Long) map.get("totalCount") > 0 && map.get("totalCount") == map.get("numberNameCount")).count()); - } - - results.put("ontologyPrincipalConceptCodeCount", new SqlSelector(schema, "SELECT COUNT(*) FROM exp.propertydescriptor WHERE principalconceptcode IS NOT NULL").getObject(Long.class)); - results.put("ontologyLookupColumnCount", new SqlSelector(schema, "SELECT COUNT(*) FROM exp.propertydescriptor WHERE concepturi = ?", OntologyService.conceptCodeConceptURI).getObject(Long.class)); - results.put("ontologyConceptSubtreeCount", new SqlSelector(schema, "SELECT COUNT(*) FROM exp.propertydescriptor WHERE conceptsubtree IS NOT NULL").getObject(Long.class)); - results.put("ontologyConceptImportColumnCount", new SqlSelector(schema, "SELECT COUNT(*) FROM exp.propertydescriptor WHERE conceptimportcolumn IS NOT NULL").getObject(Long.class)); - results.put("ontologyConceptLabelColumnCount", new SqlSelector(schema, "SELECT COUNT(*) FROM exp.propertydescriptor WHERE conceptlabelcolumn IS NOT NULL").getObject(Long.class)); - - results.put("scannableColumnCount", new SqlSelector(schema, "SELECT COUNT(*) FROM exp.propertydescriptor WHERE scannable = ?", true).getObject(Long.class)); - results.put("uniqueIdColumnCount", new SqlSelector(schema, "SELECT COUNT(*) FROM exp.propertydescriptor WHERE concepturi = ?", STORAGE_UNIQUE_ID_CONCEPT_URI).getObject(Long.class)); - results.put("sampleTypeWithUniqueIdCount", new SqlSelector(schema, """ - SELECT COUNT(DISTINCT DD.DomainURI) FROM - exp.PropertyDescriptor D\s - JOIN exp.PropertyDomain PD ON D.propertyId = PD.propertyid - JOIN exp.DomainDescriptor DD on PD.domainID = DD.domainId - WHERE D.conceptURI = ?""", STORAGE_UNIQUE_ID_CONCEPT_URI).getObject(Long.class)); - - results.put("fileColumnCount", new SqlSelector(schema, "SELECT COUNT(*) FROM exp.propertydescriptor WHERE rangeURI = ?", PropertyType.FILE_LINK.getTypeUri()).getObject(Long.class)); - results.put("sampleTypeWithFileColumnCount", new SqlSelector(schema, """ - SELECT COUNT(DISTINCT DD.DomainURI) FROM - exp.PropertyDescriptor D\s - JOIN exp.PropertyDomain PD ON D.propertyId = PD.propertyid - JOIN exp.DomainDescriptor DD on PD.domainID = DD.domainId - WHERE DD.storageSchemaName = ? AND D.rangeURI = ?""", SampleTypeDomainKind.PROVISIONED_SCHEMA_NAME, PropertyType.FILE_LINK.getTypeUri()).getObject(Long.class)); - results.put("sampleTypeWithBooleanColumnCount", new SqlSelector(schema, """ - SELECT COUNT(DISTINCT DD.DomainURI) FROM - exp.PropertyDescriptor D\s - JOIN exp.PropertyDomain PD ON D.propertyId = PD.propertyid - JOIN exp.DomainDescriptor DD on PD.domainID = DD.domainId - WHERE DD.storageSchemaName = ? AND D.rangeURI = ?""", SampleTypeDomainKind.PROVISIONED_SCHEMA_NAME, PropertyType.BOOLEAN.getTypeUri()).getObject(Long.class)); - - results.put("sampleTypeAliquotSpecificField", new SqlSelector(schema, """ - SELECT COUNT(DISTINCT D.PropertyURI) FROM - exp.PropertyDescriptor D\s - JOIN exp.PropertyDomain PD ON D.propertyId = PD.propertyid - JOIN exp.DomainDescriptor DD on PD.domainID = DD.domainId - WHERE DD.storageSchemaName = ? AND D.derivationDataScope = ?""", SampleTypeDomainKind.PROVISIONED_SCHEMA_NAME, ExpSchema.DerivationDataScopeType.ChildOnly.name()).getObject(Long.class)); - results.put("sampleTypeParentOnlyField", new SqlSelector(schema, """ - SELECT COUNT(DISTINCT D.PropertyURI) FROM - exp.PropertyDescriptor D\s - JOIN exp.PropertyDomain PD ON D.propertyId = PD.propertyid - JOIN exp.DomainDescriptor DD on PD.domainID = DD.domainId - WHERE DD.storageSchemaName = ? AND (D.derivationDataScope = ? OR D.derivationDataScope IS NULL)""", SampleTypeDomainKind.PROVISIONED_SCHEMA_NAME, ExpSchema.DerivationDataScopeType.ParentOnly.name()).getObject(Long.class)); - results.put("sampleTypeParentAndAliquotField", new SqlSelector(schema, """ - SELECT COUNT(DISTINCT D.PropertyURI) FROM - exp.PropertyDescriptor D\s - JOIN exp.PropertyDomain PD ON D.propertyId = PD.propertyid - JOIN exp.DomainDescriptor DD on PD.domainID = DD.domainId - WHERE DD.storageSchemaName = ? AND D.derivationDataScope = ?""", SampleTypeDomainKind.PROVISIONED_SCHEMA_NAME, ExpSchema.DerivationDataScopeType.All.name()).getObject(Long.class)); - - results.put("attachmentColumnCount", new SqlSelector(schema, "SELECT COUNT(*) FROM exp.propertydescriptor WHERE rangeURI = ?", PropertyType.ATTACHMENT.getTypeUri()).getObject(Long.class)); - results.put("dataClassWithAttachmentColumnCount", new SqlSelector(schema, """ - SELECT COUNT(DISTINCT DD.DomainURI) FROM - exp.PropertyDescriptor D\s - JOIN exp.PropertyDomain PD ON D.propertyId = PD.propertyid - JOIN exp.DomainDescriptor DD on PD.domainID = DD.domainId - WHERE DD.storageSchemaName = ? AND D.rangeURI = ?""", DataClassDomainKind.PROVISIONED_SCHEMA_NAME, PropertyType.ATTACHMENT.getTypeUri()).getObject(Long.class)); - results.put("dataClassWithBooleanColumnCount", new SqlSelector(schema, """ - SELECT COUNT(DISTINCT DD.DomainURI) FROM - exp.PropertyDescriptor D\s - JOIN exp.PropertyDomain PD ON D.propertyId = PD.propertyid - JOIN exp.DomainDescriptor DD on PD.domainID = DD.domainId - WHERE DD.storageSchemaName = ? AND D.rangeURI = ?""", DataClassDomainKind.PROVISIONED_SCHEMA_NAME, PropertyType.BOOLEAN.getTypeUri()).getObject(Long.class)); - - results.put("textChoiceColumnCount", new SqlSelector(schema, "SELECT COUNT(*) FROM exp.propertydescriptor WHERE concepturi = ?", TEXT_CHOICE_CONCEPT_URI).getObject(Long.class)); - results.put("multiValueTextChoiceColumnCount", new SqlSelector(schema, "SELECT COUNT(*) FROM exp.propertydescriptor WHERE rangeuri = ?", PropertyType.MULTI_CHOICE.getTypeUri()).getObject(Long.class)); - - results.put("domainsWithDateTimeColumnCount", new SqlSelector(schema, """ - SELECT COUNT(DISTINCT DD.DomainURI) FROM - exp.PropertyDescriptor D\s - JOIN exp.PropertyDomain PD ON D.propertyId = PD.propertyid - JOIN exp.DomainDescriptor DD on PD.domainID = DD.domainId - WHERE D.rangeURI = ?""", PropertyType.DATE_TIME.getTypeUri()).getObject(Long.class)); - - results.put("domainsWithDateColumnCount", new SqlSelector(schema, """ - SELECT COUNT(DISTINCT DD.DomainURI) FROM - exp.PropertyDescriptor D\s - JOIN exp.PropertyDomain PD ON D.propertyId = PD.propertyid - JOIN exp.DomainDescriptor DD on PD.domainID = DD.domainId - WHERE D.rangeURI = ?""", PropertyType.DATE.getTypeUri()).getObject(Long.class)); - - results.put("domainsWithTimeColumnCount", new SqlSelector(schema, """ - SELECT COUNT(DISTINCT DD.DomainURI) FROM - exp.PropertyDescriptor D\s - JOIN exp.PropertyDomain PD ON D.propertyId = PD.propertyid - JOIN exp.DomainDescriptor DD on PD.domainID = DD.domainId - WHERE D.rangeURI = ?""", PropertyType.TIME.getTypeUri()).getObject(Long.class)); - - results.put("maxObjectObjectId", new SqlSelector(schema, "SELECT MAX(ObjectId) FROM exp.Object").getObject(Long.class)); - results.put("maxMaterialRowId", new SqlSelector(schema, "SELECT MAX(RowId) FROM exp.Material").getObject(Long.class)); - - results.putAll(ExperimentService.get().getDomainMetrics()); - - return results; - }); - } - - ExperimentMigrationSchemaHandler handler = new ExperimentMigrationSchemaHandler(); - DatabaseMigrationService.get().registerSchemaHandler(handler); - DatabaseMigrationService.get().registerTableHandler(new MigrationTableHandler() - { - @Override - public TableInfo getTableInfo() - { - return DbSchema.get("premium", DbSchemaType.Bare).getTable("Exclusions"); - } - - @Override - public void adjustFilter(TableInfo sourceTable, SimpleFilter filter, Set containers) - { - // Include experiment runs that were copied - FilterClause includedClause = handler.getIncludedRowIdClause(sourceTable, FieldKey.fromParts("RunId")); - if (includedClause != null) - filter.addClause(includedClause); - } - }); - DatabaseMigrationService.get().registerTableHandler(new MigrationTableHandler() - { - @Override - public TableInfo getTableInfo() - { - return DbSchema.get("premium", DbSchemaType.Bare).getTable("ExclusionMaps"); - } - - @Override - public void adjustFilter(TableInfo sourceTable, SimpleFilter filter, Set containers) - { - // Include experiment runs that were copied - FilterClause includedClause = handler.getIncludedRowIdClause(sourceTable, FieldKey.fromParts("ExclusionId", "RunId")); - if (includedClause != null) - filter.addClause(includedClause); - } - }); - DatabaseMigrationService.get().registerTableHandler(new MigrationTableHandler() - { - @Override - public TableInfo getTableInfo() - { - return DbSchema.get("assayrequest", DbSchemaType.Bare).getTable("RequestRunsJunction"); - } - - @Override - public void adjustFilter(TableInfo sourceTable, SimpleFilter filter, Set containers) - { - // Include experiment runs that were copied - FilterClause includedClause = handler.getIncludedRowIdClause(sourceTable, FieldKey.fromParts("RunId")); - if (includedClause != null) - filter.addClause(includedClause); - } - }); - DatabaseMigrationService.get().registerSchemaHandler(new SampleTypeMigrationSchemaHandler()); - DataClassMigrationSchemaHandler dcHandler = new DataClassMigrationSchemaHandler(); - DatabaseMigrationService.get().registerSchemaHandler(dcHandler); - ExperimentDeleteService.setInstance(dcHandler); - } - - @Override - @NotNull - public Collection getSummary(Container c) - { - Collection list = new LinkedList<>(); - int runGroupCount = ExperimentService.get().getExperiments(c, null, false, true).size(); - if (runGroupCount > 0) - list.add(StringUtilsLabKey.pluralize(runGroupCount, "Run Group")); - - User user = HttpView.currentContext().getUser(); - - Set runTypes = ExperimentService.get().getExperimentRunTypes(c); - for (ExperimentRunType runType : runTypes) - { - if (runType == ExperimentRunType.ALL_RUNS_TYPE) - continue; - - long runCount = runType.getRunCount(user, c); - if (runCount > 0) - list.add(runCount + " runs of type " + runType.getDescription()); - } - - int dataClassCount = ExperimentService.get().getDataClasses(c, false).size(); - if (dataClassCount > 0) - list.add(dataClassCount + " Data Class" + (dataClassCount > 1 ? "es" : "")); - - int sampleTypeCount = SampleTypeService.get().getSampleTypes(c, false).size(); - if (sampleTypeCount > 0) - list.add(sampleTypeCount + " Sample Type" + (sampleTypeCount > 1 ? "s" : "")); - - return list; - } - - @Override - public @NotNull ArrayList getDetailedSummary(Container c, User user) - { - ArrayList summaries = new ArrayList<>(); - - // Assay types - long assayTypeCount = AssayService.get().getAssayProtocols(c).stream().filter(p -> p.getContainer().equals(c)).count(); - if (assayTypeCount > 0) - summaries.add(new Summary(assayTypeCount, "Assay Type")); - - // Run count - int runGroupCount = ExperimentService.get().getExperiments(c, user, false, true).size(); - if (runGroupCount > 0) - summaries.add(new Summary(runGroupCount, "Assay run")); - - // Number of Data Classes - List dataClasses = ExperimentService.get().getDataClasses(c, false); - int dataClassCount = dataClasses.size(); - if (dataClassCount > 0) - summaries.add(new Summary(dataClassCount, "Data Class")); - - ExpSchema expSchema = new ExpSchema(user, c); - - // Individual Data Class row counts - { - // The table-level container filter is set to ensure data class types are included - // that may not be defined in the target container but may have rows of data in the target container - TableInfo table = ExpSchema.TableType.DataClasses.createTable(expSchema, null, ContainerFilter.Type.CurrentPlusProjectAndShared.create(c, user)); - - // Issue 47919: The "DataCount" column is filtered to only count data in the target container - if (table instanceof ExpDataClassTableImpl tableImpl) - tableImpl.setDataCountContainerFilter(ContainerFilter.Type.Current.create(c, user)); - - Set columns = new LinkedHashSet<>(); - columns.add(ExpDataClassTable.Column.Name.name()); - columns.add(ExpDataClassTable.Column.DataCount.name()); - - Map results = new TableSelector(table, columns).getValueMap(String.class); - for (var entry : results.entrySet()) - { - long count = entry.getValue().longValue(); - if (count > 0) - summaries.add(new Summary(count, entry.getKey())); - } - } - - // Sample Types - int sampleTypeCount = SampleTypeService.get().getSampleTypes(c, false).size(); - if (sampleTypeCount > 0) - summaries.add(new Summary(sampleTypeCount, "Sample Type")); - - // Individual Sample Type row counts - { - // The table-level container filter is set to ensure data class types are included - // that may not be defined in the target container but may have rows of data in the target container - TableInfo table = ExpSchema.TableType.SampleSets.createTable(expSchema, null, ContainerFilter.Type.CurrentPlusProjectAndShared.create(c, user)); - - // Issue 51557: The "SampleCount" column is filtered to only count data in the target container - if (table instanceof ExpSampleTypeTableImpl tableImpl) - tableImpl.setSampleCountContainerFilter(ContainerFilter.Type.Current.create(c, user)); - - Set columns = new LinkedHashSet<>(); - columns.add(ExpSampleTypeTable.Column.Name.name()); - columns.add(ExpSampleTypeTable.Column.SampleCount.name()); - - Map results = new TableSelector(table, columns).getValueMap(String.class); - for (var entry : results.entrySet()) - { - long count = entry.getValue().longValue(); - if (count > 0) - { - String name = entry.getKey(); - Summary s = name.equals("MixtureBatches") - ? new Summary(count, "Batch") - : new Summary(count, name); - summaries.add(s); - } - } - } - - return summaries; - } - - @Override - public @NotNull Set> getIntegrationTests() - { - return Set.of( - DomainImpl.TestCase.class, - DomainPropertyImpl.TestCase.class, - ExpDataTableImpl.TestCase.class, - ExperimentServiceImpl.AuditDomainUriTest.class, - ExperimentServiceImpl.LineageQueryTestCase.class, - ExperimentServiceImpl.ParseInputOutputAliasTestCase.class, - ExperimentServiceImpl.TestCase.class, - ExperimentStressTest.class, - LineagePerfTest.class, - LineageTest.class, - OntologyManager.TestCase.class, - PropertyServiceImpl.TestCase.class, - SampleTypeServiceImpl.TestCase.class, - StorageNameGenerator.TestCase.class, - StorageProvisionerImpl.TestCase.class, - UniqueValueCounterTestCase.class, - XarTestPipelineJob.TestCase.class - ); - } - - @Override - public @NotNull Collection>> getIntegrationTestFactories() - { - List>> list = new ArrayList<>(super.getIntegrationTestFactories()); - list.add(new JspTestCase("/org/labkey/experiment/api/ExpDataClassDataTestCase.jsp")); - list.add(new JspTestCase("/org/labkey/experiment/api/ExpSampleTypeTestCase.jsp")); - return list; - } - - @Override - public @NotNull Set> getUnitTests() - { - return Set.of( - GraphAlgorithms.TestCase.class, - LSIDRelativizer.TestCase.class, - Lsid.TestCase.class, - LsidUtils.TestCase.class, - PropertyController.TestCase.class, - Quantity.TestCase.class, - Unit.TestCase.class - ); - } - - @Override - @NotNull - public Collection getSchemaNames() - { - return List.of( - ExpSchema.SCHEMA_NAME, - DataClassDomainKind.PROVISIONED_SCHEMA_NAME, - SampleTypeDomainKind.PROVISIONED_SCHEMA_NAME - ); - } - - @NotNull - @Override - public Collection getProvisionedSchemaNames() - { - return PageFlowUtil.set(DataClassDomainKind.PROVISIONED_SCHEMA_NAME, SampleTypeDomainKind.PROVISIONED_SCHEMA_NAME); - } -} +/* + * Copyright (c) 2008-2019 LabKey Corporation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.labkey.experiment; + +import org.apache.commons.lang3.math.NumberUtils; +import org.jetbrains.annotations.NotNull; +import org.jetbrains.annotations.Nullable; +import org.labkey.api.admin.FolderSerializationRegistry; +import org.labkey.api.assay.AssayProvider; +import org.labkey.api.assay.AssayService; +import org.labkey.api.attachments.AttachmentService; +import org.labkey.api.audit.AuditLogService; +import org.labkey.api.audit.SampleTimelineAuditEvent; +import org.labkey.api.collections.LongHashMap; +import org.labkey.api.data.Container; +import org.labkey.api.data.ContainerFilter; +import org.labkey.api.data.ContainerManager; +import org.labkey.api.data.CoreSchema; +import org.labkey.api.data.DbSchema; +import org.labkey.api.data.DbSchemaType; +import org.labkey.api.data.JdbcType; +import org.labkey.api.data.NameGenerator; +import org.labkey.api.data.SQLFragment; +import org.labkey.api.data.SimpleFilter; +import org.labkey.api.data.SimpleFilter.FilterClause; +import org.labkey.api.data.SqlSelector; +import org.labkey.api.data.TableInfo; +import org.labkey.api.data.TableSelector; +import org.labkey.api.data.UpgradeCode; +import org.labkey.api.defaults.DefaultValueService; +import org.labkey.api.exp.ExperimentException; +import org.labkey.api.exp.ExperimentRunType; +import org.labkey.api.exp.Lsid; +import org.labkey.api.exp.OntologyManager; +import org.labkey.api.exp.PropertyType; +import org.labkey.api.exp.api.DefaultExperimentDataHandler; +import org.labkey.api.exp.api.ExpData; +import org.labkey.api.exp.api.ExpDataClass; +import org.labkey.api.exp.api.ExpLineageService; +import org.labkey.api.exp.api.ExpMaterial; +import org.labkey.api.exp.api.ExpProtocol; +import org.labkey.api.exp.api.ExpProtocolAttachmentType; +import org.labkey.api.exp.api.ExpRunAttachmentType; +import org.labkey.api.exp.api.ExpSampleType; +import org.labkey.api.exp.api.ExperimentJSONConverter; +import org.labkey.api.exp.api.ExperimentService; +import org.labkey.api.exp.api.FilterProtocolInputCriteria; +import org.labkey.api.exp.api.SampleTypeDomainKind; +import org.labkey.api.exp.api.SampleTypeService; +import org.labkey.api.exp.api.StorageProvisioner; +import org.labkey.api.exp.property.DomainAuditProvider; +import org.labkey.api.exp.property.DomainPropertyAuditProvider; +import org.labkey.api.exp.property.ExperimentProperty; +import org.labkey.api.exp.property.PropertyService; +import org.labkey.api.exp.property.SystemProperty; +import org.labkey.api.exp.query.ExpDataClassTable; +import org.labkey.api.exp.query.ExpSampleTypeTable; +import org.labkey.api.exp.query.ExpSchema; +import org.labkey.api.exp.query.SamplesSchema; +import org.labkey.api.exp.xar.LSIDRelativizer; +import org.labkey.api.exp.xar.LsidUtils; +import org.labkey.api.files.FileContentService; +import org.labkey.api.files.TableUpdaterFileListener; +import org.labkey.api.migration.DatabaseMigrationService; +import org.labkey.api.migration.ExperimentDeleteService; +import org.labkey.api.migration.MigrationTableHandler; +import org.labkey.api.module.ModuleContext; +import org.labkey.api.module.ModuleLoader; +import org.labkey.api.module.SpringModule; +import org.labkey.api.module.Summary; +import org.labkey.api.ontology.OntologyService; +import org.labkey.api.ontology.Quantity; +import org.labkey.api.ontology.Unit; +import org.labkey.api.pipeline.PipelineService; +import org.labkey.api.query.FieldKey; +import org.labkey.api.query.FilteredTable; +import org.labkey.api.query.QueryService; +import org.labkey.api.query.UserSchema; +import org.labkey.api.search.SearchService; +import org.labkey.api.security.User; +import org.labkey.api.security.roles.RoleManager; +import org.labkey.api.settings.AppProps; +import org.labkey.api.settings.OptionalFeatureService; +import org.labkey.api.usageMetrics.UsageMetricsService; +import org.labkey.api.util.GUID; +import org.labkey.api.util.JspTestCase; +import org.labkey.api.util.PageFlowUtil; +import org.labkey.api.util.StringUtilsLabKey; +import org.labkey.api.util.SystemMaintenance; +import org.labkey.api.view.AlwaysAvailableWebPartFactory; +import org.labkey.api.view.BaseWebPartFactory; +import org.labkey.api.view.HttpView; +import org.labkey.api.view.JspView; +import org.labkey.api.view.Portal; +import org.labkey.api.view.ViewContext; +import org.labkey.api.view.WebPartFactory; +import org.labkey.api.view.WebPartView; +import org.labkey.api.view.template.WarningService; +import org.labkey.api.vocabulary.security.DesignVocabularyPermission; +import org.labkey.api.webdav.WebdavResource; +import org.labkey.api.webdav.WebdavService; +import org.labkey.experiment.api.DataClassDomainKind; +import org.labkey.experiment.api.ExpDataClassImpl; +import org.labkey.experiment.api.ExpDataClassTableImpl; +import org.labkey.experiment.api.ExpDataClassType; +import org.labkey.experiment.api.ExpDataImpl; +import org.labkey.experiment.api.ExpDataTableImpl; +import org.labkey.experiment.api.ExpMaterialImpl; +import org.labkey.experiment.api.ExpProtocolImpl; +import org.labkey.experiment.api.ExpSampleTypeImpl; +import org.labkey.experiment.api.ExpSampleTypeTableImpl; +import org.labkey.experiment.api.ExperimentServiceImpl; +import org.labkey.experiment.api.ExperimentStressTest; +import org.labkey.experiment.api.GraphAlgorithms; +import org.labkey.experiment.api.LineageTest; +import org.labkey.experiment.api.LogDataType; +import org.labkey.experiment.api.Protocol; +import org.labkey.experiment.api.SampleTypeServiceImpl; +import org.labkey.experiment.api.SampleTypeUpdateServiceDI; +import org.labkey.experiment.api.UniqueValueCounterTestCase; +import org.labkey.experiment.api.VocabularyDomainKind; +import org.labkey.experiment.api.data.ChildOfCompareType; +import org.labkey.experiment.api.data.ChildOfMethod; +import org.labkey.experiment.api.data.LineageCompareType; +import org.labkey.experiment.api.data.ParentOfCompareType; +import org.labkey.experiment.api.data.ParentOfMethod; +import org.labkey.experiment.api.property.DomainImpl; +import org.labkey.experiment.api.property.DomainPropertyImpl; +import org.labkey.experiment.api.property.LengthValidator; +import org.labkey.experiment.api.property.LookupValidator; +import org.labkey.experiment.api.property.PropertyServiceImpl; +import org.labkey.experiment.api.property.RangeValidator; +import org.labkey.experiment.api.property.RegExValidator; +import org.labkey.experiment.api.property.StorageNameGenerator; +import org.labkey.experiment.api.property.StorageProvisionerImpl; +import org.labkey.experiment.api.property.TextChoiceValidator; +import org.labkey.experiment.controllers.exp.ExperimentController; +import org.labkey.experiment.controllers.property.PropertyController; +import org.labkey.experiment.defaults.DefaultValueServiceImpl; +import org.labkey.experiment.lineage.ExpLineageServiceImpl; +import org.labkey.experiment.lineage.LineagePerfTest; +import org.labkey.experiment.pipeline.ExperimentPipelineProvider; +import org.labkey.experiment.pipeline.XarTestPipelineJob; +import org.labkey.experiment.samples.DataClassFolderImporter; +import org.labkey.experiment.samples.DataClassFolderWriter; +import org.labkey.experiment.samples.SampleStatusFolderImporter; +import org.labkey.experiment.samples.SampleTimelineAuditProvider; +import org.labkey.experiment.samples.SampleTypeFolderImporter; +import org.labkey.experiment.samples.SampleTypeFolderWriter; +import org.labkey.experiment.security.DataClassDesignerRole; +import org.labkey.experiment.security.SampleTypeDesignerRole; +import org.labkey.experiment.types.TypesController; +import org.labkey.experiment.xar.FolderXarImporterFactory; +import org.labkey.experiment.xar.FolderXarWriterFactory; + +import java.util.ArrayList; +import java.util.Collection; +import java.util.Collections; +import java.util.HashMap; +import java.util.HashSet; +import java.util.LinkedHashSet; +import java.util.LinkedList; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.function.Supplier; +import java.util.stream.Collectors; + +import static org.labkey.api.data.ColumnRenderPropertiesImpl.STORAGE_UNIQUE_ID_CONCEPT_URI; +import static org.labkey.api.data.ColumnRenderPropertiesImpl.TEXT_CHOICE_CONCEPT_URI; +import static org.labkey.api.exp.api.ExperimentService.MODULE_NAME; + +public class ExperimentModule extends SpringModule +{ + private static final String SAMPLE_TYPE_WEB_PART_NAME = "Sample Types"; + private static final String PROTOCOL_WEB_PART_NAME = "Protocols"; + + public static final String AMOUNT_AND_UNIT_UPGRADE_PROP = "AmountAndUnitAudit"; + public static final String TRANSACTION_ID_PROP = "AuditTransactionId"; + public static final String AUDIT_COUNT_PROP = "AuditRecordCount"; + public static final String EXPERIMENT_RUN_WEB_PART_NAME = "Experiment Runs"; + + @Override + public String getName() + { + return MODULE_NAME; + } + + @Override + public Double getSchemaVersion() + { + return 26.001; + } + + @Nullable + @Override + public UpgradeCode getUpgradeCode() + { + return new ExperimentUpgradeCode(); + } + + @Override + protected void init() + { + addController("experiment", ExperimentController.class); + addController("experiment-types", TypesController.class); + addController("property", PropertyController.class); + ExperimentService.setInstance(new ExperimentServiceImpl()); + SampleTypeService.setInstance(new SampleTypeServiceImpl()); + DefaultValueService.setInstance(new DefaultValueServiceImpl()); + StorageProvisioner.setInstance(StorageProvisionerImpl.get()); + ExpLineageService.setInstance(new ExpLineageServiceImpl()); + + PropertyServiceImpl propertyServiceImpl = new PropertyServiceImpl(); + PropertyService.setInstance(propertyServiceImpl); + UsageMetricsService.get().registerUsageMetrics(getName(), propertyServiceImpl); + + UsageMetricsService.get().registerUsageMetrics(getName(), FileLinkMetricsProvider.getInstance()); + + ExperimentProperty.register(); + SamplesSchema.register(this); + ExpSchema.register(this); + + PropertyService.get().registerDomainKind(new SampleTypeDomainKind()); + PropertyService.get().registerDomainKind(new DataClassDomainKind()); + PropertyService.get().registerDomainKind(new VocabularyDomainKind()); + + QueryService.get().addCompareType(new ChildOfCompareType()); + QueryService.get().addCompareType(new ParentOfCompareType()); + QueryService.get().addCompareType(new LineageCompareType()); + QueryService.get().registerMethod(ChildOfMethod.NAME, new ChildOfMethod(), JdbcType.BOOLEAN, 2, 3); + QueryService.get().registerMethod(ParentOfMethod.NAME, new ParentOfMethod(), JdbcType.BOOLEAN, 2, 3); + QueryService.get().addQueryListener(new ExperimentQueryChangeListener()); + QueryService.get().addQueryListener(new PropertyQueryChangeListener()); + + PropertyService.get().registerValidatorKind(new RegExValidator()); + PropertyService.get().registerValidatorKind(new RangeValidator()); + PropertyService.get().registerValidatorKind(new LookupValidator()); + PropertyService.get().registerValidatorKind(new LengthValidator()); + PropertyService.get().registerValidatorKind(new TextChoiceValidator()); + + ExperimentService.get().registerExperimentDataHandler(new DefaultExperimentDataHandler()); + ExperimentService.get().registerProtocolInputCriteria(new FilterProtocolInputCriteria.Factory()); + ExperimentService.get().registerNameExpressionType("sampletype", "exp", "MaterialSource", "nameexpression"); + ExperimentService.get().registerNameExpressionType("aliquots", "exp", "MaterialSource", "aliquotnameexpression"); + ExperimentService.get().registerNameExpressionType("dataclass", "exp", "DataClass", "nameexpression"); + + OptionalFeatureService.get().addExperimentalFeatureFlag(AppProps.EXPERIMENTAL_RESOLVE_PROPERTY_URI_COLUMNS, "Resolve property URIs as columns on experiment tables", + "If a column is not found on an experiment table, attempt to resolve the column name as a Property URI and add it as a property column", false); + if (CoreSchema.getInstance().getSqlDialect().isSqlServer()) + { + OptionalFeatureService.get().addExperimentalFeatureFlag(NameGenerator.EXPERIMENTAL_WITH_COUNTER, "Use strict incremental withCounter and rootSampleCount expression", + "When withCounter or rootSampleCount is used in name expression, make sure the count increments one-by-one and does not jump.", true); + } + else + { + OptionalFeatureService.get().addExperimentalFeatureFlag(NameGenerator.EXPERIMENTAL_ALLOW_GAP_COUNTER, "Allow gap with withCounter and rootSampleCount expression", + "Check this option if gaps in the count generated by withCounter or rootSampleCount name expression are allowed.", true); + OptionalFeatureService.get().addExperimentalFeatureFlag(AppProps.MULTI_VALUE_TEXT_CHOICE, "Allow multi-value Text Choice properties", + "Support selecting more than one values for text choice fields", false); + } + OptionalFeatureService.get().addExperimentalFeatureFlag(AppProps.QUANTITY_COLUMN_SUFFIX_TESTING, "Quantity column suffix testing", + "If a column name contains a \"__\" suffix, this feature allows for testing it as a Quantity display column", false); + OptionalFeatureService.get().addExperimentalFeatureFlag(ExperimentService.EXPERIMENTAL_FEATURE_FROM_EXPANCESTORS, "SQL syntax: 'FROM EXPANCESTORS()'", + "Support for querying lineage of experiment objects", false); + OptionalFeatureService.get().addExperimentalFeatureFlag(SampleTypeUpdateServiceDI.EXPERIMENTAL_FEATURE_ALLOW_ROW_ID_SAMPLE_MERGE, "Allow RowId to be accepted when merging samples", + "If the incoming data includes a RowId column we will allow the column but ignore it's values.", false); + + RoleManager.registerPermission(new DesignVocabularyPermission(), true); + RoleManager.registerRole(new SampleTypeDesignerRole()); + RoleManager.registerRole(new DataClassDesignerRole()); + + AttachmentService.get().registerAttachmentParentType(ExpRunAttachmentType.get()); + AttachmentService.get().registerAttachmentParentType(ExpProtocolAttachmentType.get()); + + WebdavService.get().addExpDataProvider((path, container) -> ExperimentService.get().getAllExpDataByURL(path, container)); + ExperimentService.get().registerObjectReferencer(ExperimentServiceImpl.get()); + + addModuleProperty(new LineageMaximumDepthModuleProperty(this)); + WarningService.get().register(new ExperimentWarningProvider()); + } + + @Override + public boolean hasScripts() + { + return true; + } + + @Override + @NotNull + protected Collection createWebPartFactories() + { + List result = new ArrayList<>(); + + BaseWebPartFactory runGroupsFactory = new BaseWebPartFactory(RunGroupWebPart.WEB_PART_NAME, WebPartFactory.LOCATION_BODY, WebPartFactory.LOCATION_RIGHT) + { + @Override + public WebPartView getWebPartView(@NotNull ViewContext portalCtx, @NotNull Portal.WebPart webPart) + { + return new RunGroupWebPart(portalCtx, WebPartFactory.LOCATION_RIGHT.equalsIgnoreCase(webPart.getLocation()), webPart); + } + }; + runGroupsFactory.addLegacyNames("Experiments", "Experiment", "Experiment Navigator", "Narrow Experiments"); + result.add(runGroupsFactory); + + BaseWebPartFactory runTypesFactory = new BaseWebPartFactory(RunTypeWebPart.WEB_PART_NAME, WebPartFactory.LOCATION_BODY, WebPartFactory.LOCATION_RIGHT) + { + @Override + public WebPartView getWebPartView(@NotNull ViewContext portalCtx, @NotNull Portal.WebPart webPart) + { + return new RunTypeWebPart(); + } + }; + result.add(runTypesFactory); + + result.add(new ExperimentRunWebPartFactory()); + BaseWebPartFactory sampleTypeFactory = new BaseWebPartFactory(SAMPLE_TYPE_WEB_PART_NAME, WebPartFactory.LOCATION_BODY, WebPartFactory.LOCATION_RIGHT) + { + @Override + public WebPartView getWebPartView(@NotNull ViewContext portalCtx, @NotNull Portal.WebPart webPart) + { + return new SampleTypeWebPart(WebPartFactory.LOCATION_RIGHT.equalsIgnoreCase(webPart.getLocation()), portalCtx); + } + }; + sampleTypeFactory.addLegacyNames("Narrow Sample Sets", "Sample Sets"); + result.add(sampleTypeFactory); + result.add(new AlwaysAvailableWebPartFactory("Samples Menu", false, false, WebPartFactory.LOCATION_MENUBAR) { + @Override + public WebPartView getWebPartView(@NotNull ViewContext portalCtx, @NotNull Portal.WebPart webPart) + { + WebPartView view = new JspView<>("/org/labkey/experiment/samplesAndAnalytes.jsp", webPart); + view.setTitle("Samples"); + return view; + } + }); + + result.add(new AlwaysAvailableWebPartFactory("Data Classes", false, false, WebPartFactory.LOCATION_BODY, WebPartFactory.LOCATION_RIGHT) { + @Override + public WebPartView getWebPartView(@NotNull ViewContext portalCtx, @NotNull Portal.WebPart webPart) + { + return new DataClassWebPart(WebPartFactory.LOCATION_RIGHT.equalsIgnoreCase(webPart.getLocation()), portalCtx, webPart); + } + }); + + BaseWebPartFactory narrowProtocolFactory = new BaseWebPartFactory(PROTOCOL_WEB_PART_NAME, WebPartFactory.LOCATION_RIGHT) + { + @Override + public WebPartView getWebPartView(@NotNull ViewContext portalCtx, @NotNull Portal.WebPart webPart) + { + return new ProtocolWebPart(WebPartFactory.LOCATION_RIGHT.equalsIgnoreCase(webPart.getLocation()), portalCtx); + } + }; + narrowProtocolFactory.addLegacyNames("Narrow Protocols"); + result.add(narrowProtocolFactory); + + return result; + } + + private void addDataResourceResolver(String categoryName) + { + SearchService.get().addResourceResolver(categoryName, new SearchService.ResourceResolver() + { + @Override + public WebdavResource resolve(@NotNull String resourceIdentifier) + { + ExpDataImpl data = ExpDataImpl.fromDocumentId(resourceIdentifier); + if (data == null) + return null; + + return data.createIndexDocument(null); + } + + @Override + public Map getCustomSearchJson(User user, @NotNull String resourceIdentifier) + { + ExpDataImpl data = ExpDataImpl.fromDocumentId(resourceIdentifier); + if (data == null) + return null; + + return ExperimentJSONConverter.serializeData(data, user, ExperimentJSONConverter.DEFAULT_SETTINGS).toMap(); + } + + @Override + public Map> getCustomSearchJsonMap(User user, @NotNull Collection resourceIdentifiers) + { + Map idDataMap = ExpDataImpl.fromDocumentIds(resourceIdentifiers); + if (idDataMap == null) + return null; + + Map> searchJsonMap = new HashMap<>(); + for (String resourceIdentifier : idDataMap.keySet()) + searchJsonMap.put(resourceIdentifier, ExperimentJSONConverter.serializeData(idDataMap.get(resourceIdentifier), user, ExperimentJSONConverter.DEFAULT_SETTINGS).toMap()); + return searchJsonMap; + } + }); + } + + private void addDataClassResourceResolver(String categoryName) + { + SearchService.get().addResourceResolver(categoryName, new SearchService.ResourceResolver(){ + @Override + public Map getCustomSearchJson(User user, @NotNull String resourceIdentifier) + { + int rowId = NumberUtils.toInt(resourceIdentifier.replace(categoryName + ":", "")); + if (rowId == 0) + return null; + + ExpDataClass dataClass = ExperimentService.get().getDataClass(rowId); + if (dataClass == null) + return null; + + Map properties = ExperimentJSONConverter.serializeExpObject(dataClass, null, ExperimentJSONConverter.DEFAULT_SETTINGS, user).toMap(); + + //Need to map to proper Icon + properties.put("type", "dataClass" + (dataClass.getCategory() != null ? ":" + dataClass.getCategory() : "")); + + return properties; + } + }); + } + + private void addSampleTypeResourceResolver(String categoryName) + { + SearchService.get().addResourceResolver(categoryName, new SearchService.ResourceResolver(){ + @Override + public Map getCustomSearchJson(User user, @NotNull String resourceIdentifier) + { + int rowId = NumberUtils.toInt(resourceIdentifier.replace(categoryName + ":", "")); + if (rowId == 0) + return null; + + ExpSampleType sampleType = SampleTypeService.get().getSampleType(rowId); + if (sampleType == null) + return null; + + Map properties = ExperimentJSONConverter.serializeExpObject(sampleType, null, ExperimentJSONConverter.DEFAULT_SETTINGS, user).toMap(); + + //Need to map to proper Icon + properties.put("type", "sampleSet"); + + return properties; + } + }); + } + + private void addSampleResourceResolver(String categoryName) + { + SearchService.get().addResourceResolver(categoryName, new SearchService.ResourceResolver(){ + @Override + public Map getCustomSearchJson(User user, @NotNull String resourceIdentifier) + { + int rowId = NumberUtils.toInt(resourceIdentifier.replace(categoryName + ":", "")); + if (rowId == 0) + return null; + + ExpMaterial material = ExperimentService.get().getExpMaterial(rowId); + if (material == null) + return null; + + return ExperimentJSONConverter.serializeMaterial(material, user, ExperimentJSONConverter.DEFAULT_SETTINGS).toMap(); + } + + @Override + public Map> getCustomSearchJsonMap(User user, @NotNull Collection resourceIdentifiers) + { + Set rowIds = new HashSet<>(); + Map rowIdIdentifierMap = new LongHashMap<>(); + for (String resourceIdentifier : resourceIdentifiers) + { + long rowId = NumberUtils.toLong(resourceIdentifier.replace(categoryName + ":", "")); + if (rowId != 0) + { + rowIds.add(rowId); + rowIdIdentifierMap.put(rowId, resourceIdentifier); + } + } + + Map> searchJsonMap = new HashMap<>(); + for (ExpMaterial material : ExperimentService.get().getExpMaterials(rowIds)) + { + searchJsonMap.put( + rowIdIdentifierMap.get(material.getRowId()), + ExperimentJSONConverter.serializeMaterial(material, user, ExperimentJSONConverter.DEFAULT_SETTINGS).toMap() + ); + } + + return searchJsonMap; + } + }); + } + + @Override + protected void startupAfterSpringConfig(ModuleContext moduleContext) + { + SearchService ss = SearchService.get(); +// ss.addSearchCategory(OntologyManager.conceptCategory); + ss.addSearchCategory(ExpSampleTypeImpl.searchCategory); + ss.addSearchCategory(ExpSampleTypeImpl.mediaSearchCategory); + ss.addSearchCategory(ExpMaterialImpl.searchCategory); + ss.addSearchCategory(ExpMaterialImpl.mediaSearchCategory); + ss.addSearchCategory(ExpDataClassImpl.SEARCH_CATEGORY); + ss.addSearchCategory(ExpDataClassImpl.MEDIA_SEARCH_CATEGORY); + ss.addSearchCategory(ExpDataImpl.expDataCategory); + ss.addSearchCategory(ExpDataImpl.expMediaDataCategory); + ss.addSearchResultTemplate(new ExpDataImpl.DataSearchResultTemplate()); + addDataResourceResolver(ExpDataImpl.expDataCategory.getName()); + addDataResourceResolver(ExpDataImpl.expMediaDataCategory.getName()); + addDataClassResourceResolver(ExpDataClassImpl.SEARCH_CATEGORY.getName()); + addDataClassResourceResolver(ExpDataClassImpl.MEDIA_SEARCH_CATEGORY.getName()); + addSampleTypeResourceResolver(ExpSampleTypeImpl.searchCategory.getName()); + addSampleTypeResourceResolver(ExpSampleTypeImpl.mediaSearchCategory.getName()); + addSampleResourceResolver(ExpMaterialImpl.searchCategory.getName()); + addSampleResourceResolver(ExpMaterialImpl.mediaSearchCategory.getName()); + ss.addDocumentProvider(ExperimentServiceImpl.get()); + + PipelineService.get().registerPipelineProvider(new ExperimentPipelineProvider(this)); + ExperimentService.get().registerExperimentRunTypeSource(container -> Collections.singleton(ExperimentRunType.ALL_RUNS_TYPE)); + ExperimentService.get().registerDataType(new LogDataType()); + + AuditLogService.get().registerAuditType(new DomainAuditProvider()); + AuditLogService.get().registerAuditType(new DomainPropertyAuditProvider()); + AuditLogService.get().registerAuditType(new ExperimentAuditProvider()); + AuditLogService.get().registerAuditType(new SampleTypeAuditProvider()); + AuditLogService.get().registerAuditType(new SampleTimelineAuditProvider()); + + FileContentService fileContentService = FileContentService.get(); + if (null != fileContentService) + { + fileContentService.addFileListener(new ExpDataFileListener()); + fileContentService.addFileListener(new TableUpdaterFileListener(ExperimentService.get().getTinfoExperimentRun(), "FilePathRoot", TableUpdaterFileListener.Type.fileRootPath, "RowId")); + fileContentService.addFileListener(new FileLinkFileListener()); + } + ContainerManager.addContainerListener(new ContainerManager.ContainerListener() + { + @Override + public void containerDeleted(Container c, User user) + { + try + { + ExperimentService.get().deleteAllExpObjInContainer(c, user); + } + catch (ExperimentException ee) + { + throw new RuntimeException(ee); + } + } + }, + // This is in the Last group because when a container is deleted, + // the Experiment listener needs to be called after the Study listener, + // because Study needs the metadata held by Experiment to delete properly. + // but it should be before the CoreContainerListener + ContainerManager.ContainerListener.Order.Last); + + if (ModuleLoader.getInstance().shouldInsertData()) + SystemProperty.registerProperties(); + + FolderSerializationRegistry folderRegistry = FolderSerializationRegistry.get(); + if (null != folderRegistry) + { + folderRegistry.addFactories(new FolderXarWriterFactory(), new FolderXarImporterFactory()); + folderRegistry.addWriterFactory(new SampleTypeFolderWriter.SampleTypeDesignWriter.Factory()); + folderRegistry.addWriterFactory(new SampleTypeFolderWriter.SampleTypeDataWriter.Factory()); + folderRegistry.addWriterFactory(new DataClassFolderWriter.DataClassDesignWriter.Factory()); + folderRegistry.addWriterFactory(new DataClassFolderWriter.DataClassDataWriter.Factory()); + folderRegistry.addImportFactory(new SampleTypeFolderImporter.Factory()); + folderRegistry.addImportFactory(new DataClassFolderImporter.Factory()); + folderRegistry.addImportFactory(new SampleStatusFolderImporter.Factory()); + } + + AttachmentService.get().registerAttachmentParentType(ExpDataClassType.get()); + + WebdavService.get().addProvider(new ScriptsResourceProvider()); + + SystemMaintenance.addTask(new FileLinkMetricsMaintenanceTask()); + + UsageMetricsService svc = UsageMetricsService.get(); + if (null != svc) + { + svc.registerUsageMetrics(getName(), () -> { + Map results = new HashMap<>(); + + DbSchema schema = ExperimentService.get().getSchema(); + if (AssayService.get() != null) + { + Map assayMetrics = new HashMap<>(); + SQLFragment baseRunSQL = new SQLFragment("SELECT COUNT(*) FROM ").append(ExperimentService.get().getTinfoExperimentRun(), "r").append(" WHERE lsid LIKE ?"); + SQLFragment baseProtocolSQL = new SQLFragment("SELECT * FROM ").append(ExperimentService.get().getTinfoProtocol(), "p").append(" WHERE lsid LIKE ? AND ApplicationType = ?"); + for (AssayProvider assayProvider : AssayService.get().getAssayProviders()) + { + Map protocolMetrics = new HashMap<>(); + + // Run count across all assay designs of this type + SQLFragment runSQL = new SQLFragment(baseRunSQL); + runSQL.add(Lsid.namespaceLikeString(assayProvider.getRunLSIDPrefix())); + protocolMetrics.put("runCount", new SqlSelector(schema, runSQL).getObject(Long.class)); + + // Number of assay designs of this type + SQLFragment protocolSQL = new SQLFragment(baseProtocolSQL); + protocolSQL.add(assayProvider.getProtocolPattern()); + protocolSQL.add(ExpProtocol.ApplicationType.ExperimentRun.toString()); + List protocols = new SqlSelector(schema, protocolSQL).getArrayList(Protocol.class); + protocolMetrics.put("protocolCount", protocols.size()); + + List wrappedProtocols = protocols.stream().map(ExpProtocolImpl::new).collect(Collectors.toList()); + + protocolMetrics.put("resultRowCount", assayProvider.getResultRowCount(wrappedProtocols)); + + // Primary implementation class + protocolMetrics.put("implementingClass", assayProvider.getClass()); + + assayMetrics.put(assayProvider.getName(), protocolMetrics); + } + assayMetrics.put("autoLinkedAssayCount", new SqlSelector(schema, "SELECT COUNT(*) FROM exp.protocol EP JOIN exp.objectPropertiesView OP ON EP.lsid = OP.objecturi WHERE OP.propertyuri = 'terms.labkey.org#AutoCopyTargetContainer'").getObject(Long.class)); + assayMetrics.put("protocolsWithTransformScriptCount", new SqlSelector(schema, "SELECT COUNT(*) FROM exp.protocol EP JOIN exp.objectPropertiesView OP ON EP.lsid = OP.objecturi WHERE OP.name = 'TransformScript' AND status = 'Active'").getObject(Long.class)); + assayMetrics.put("protocolsWithTransformScriptRunOnEditCount", new SqlSelector(schema, "SELECT COUNT(*) FROM exp.protocol EP JOIN exp.objectPropertiesView OP ON EP.lsid = OP.objecturi WHERE OP.name = 'TransformScript' AND status = 'Active' AND OP.stringvalue LIKE '%\"INSERT\"%'").getObject(Long.class)); + assayMetrics.put("protocolsWithTransformScriptRunOnImportCount", new SqlSelector(schema, "SELECT COUNT(*) FROM exp.protocol EP JOIN exp.objectPropertiesView OP ON EP.lsid = OP.objecturi WHERE OP.name = 'TransformScript' AND status = 'Active' AND OP.stringvalue LIKE '%\"INSERT\"%'").getObject(Long.class)); + + assayMetrics.put("standardAssayWithPlateSupportCount", new SqlSelector(schema, "SELECT COUNT(*) FROM exp.protocol EP JOIN exp.objectPropertiesView OP ON EP.lsid = OP.objecturi WHERE OP.name = 'PlateMetadata' AND floatValue = 1").getObject(Long.class)); + SQLFragment runsWithPlateSQL = new SQLFragment(""" + SELECT COUNT(*) FROM exp.experimentrun r + INNER JOIN exp.object o ON o.objectUri = r.lsid + INNER JOIN exp.objectproperty op ON op.objectId = o.objectId + WHERE op.propertyid IN ( + SELECT propertyid FROM exp.propertydescriptor WHERE name = ? AND lookupquery = ? + )"""); + assayMetrics.put("standardAssayRunsWithPlateTemplate", new SqlSelector(schema, new SQLFragment(runsWithPlateSQL).add("PlateTemplate").add("PlateTemplate")).getObject(Long.class)); + assayMetrics.put("standardAssayRunsWithPlateSet", new SqlSelector(schema, new SQLFragment(runsWithPlateSQL).add("PlateSet").add("PlateSet")).getObject(Long.class)); + + assayMetrics.put("assayRunsFileColumnCount", new SqlSelector(schema, """ + SELECT COUNT(DISTINCT DD.DomainURI) FROM + exp.PropertyDescriptor D\s + JOIN exp.PropertyDomain PD ON D.propertyId = PD.propertyid + JOIN exp.DomainDescriptor DD on PD.domainID = DD.domainId + WHERE DD.domainUri LIKE ? AND D.rangeURI = ?""", "urn:lsid:%:" + ExpProtocol.AssayDomainTypes.Run.getPrefix() + ".%", PropertyType.FILE_LINK.getTypeUri()).getObject(Long.class)); + + assayMetrics.put("assayResultsFileColumnCount", new SqlSelector(schema, """ + SELECT COUNT(DISTINCT DD.DomainURI) FROM + exp.PropertyDescriptor D\s + JOIN exp.PropertyDomain PD ON D.propertyId = PD.propertyid + JOIN exp.DomainDescriptor DD on PD.domainID = DD.domainId + WHERE DD.domainUri LIKE ? AND D.rangeURI = ?""", "urn:lsid:%:" + ExpProtocol.AssayDomainTypes.Result.getPrefix() + ".%", PropertyType.FILE_LINK.getTypeUri()).getObject(Long.class)); + + Map sampleLookupCountMetrics = new HashMap<>(); + SQLFragment baseAssaySampleLookupSQL = new SQLFragment("SELECT COUNT(*) FROM exp.propertydescriptor WHERE (lookupschema = 'samples' OR (lookupschema = 'exp' AND lookupquery = 'Materials')) AND propertyuri LIKE ?"); + + SQLFragment batchAssaySampleLookupSQL = new SQLFragment(baseAssaySampleLookupSQL); + batchAssaySampleLookupSQL.add("urn:lsid:%:" + ExpProtocol.AssayDomainTypes.Batch.getPrefix() + ".%"); + sampleLookupCountMetrics.put("batchDomain", new SqlSelector(schema, batchAssaySampleLookupSQL).getObject(Long.class)); + + SQLFragment runAssaySampleLookupSQL = new SQLFragment(baseAssaySampleLookupSQL); + runAssaySampleLookupSQL.add("urn:lsid:%:" + ExpProtocol.AssayDomainTypes.Run.getPrefix() + ".%"); + sampleLookupCountMetrics.put("runDomain", new SqlSelector(schema, runAssaySampleLookupSQL).getObject(Long.class)); + + SQLFragment resultAssaySampleLookupSQL = new SQLFragment(baseAssaySampleLookupSQL); + resultAssaySampleLookupSQL.add("urn:lsid:%:" + ExpProtocol.AssayDomainTypes.Result.getPrefix() + ".%"); + sampleLookupCountMetrics.put("resultDomain", new SqlSelector(schema, resultAssaySampleLookupSQL).getObject(Long.class)); + + SQLFragment resultAssayMultipleSampleLookupSQL = new SQLFragment( + """ + SELECT COUNT(*) FROM ( + SELECT PD.domainid, COUNT(*) AS PropCount + FROM exp.propertydescriptor D + JOIN exp.PropertyDomain PD ON D.propertyId = PD.propertyid + WHERE (lookupschema = 'samples' OR (lookupschema = 'exp' AND lookupquery = 'Materials')) + AND propertyuri LIKE ? + GROUP BY PD.domainid + ) X WHERE X.PropCount > 1""" + ); + resultAssayMultipleSampleLookupSQL.add("urn:lsid:%:" + ExpProtocol.AssayDomainTypes.Result.getPrefix() + ".%"); + sampleLookupCountMetrics.put("resultDomainWithMultiple", new SqlSelector(schema, resultAssayMultipleSampleLookupSQL).getObject(Long.class)); + + assayMetrics.put("sampleLookupCount", sampleLookupCountMetrics); + + + // Putting these metrics at the same level as the other BooleanColumnCount metrics (e.g., sampleTypeWithBooleanColumnCount) + results.put("assayResultWithBooleanColumnCount", new SqlSelector(schema, """ + SELECT COUNT(DISTINCT DD.DomainURI) FROM + exp.PropertyDescriptor D\s + JOIN exp.PropertyDomain PD ON D.propertyId = PD.propertyid + JOIN exp.DomainDescriptor DD on PD.domainID = DD.domainId + WHERE D.propertyURI LIKE ? AND D.rangeURI = ?""", "urn:lsid:%:" + ExpProtocol.AssayDomainTypes.Result.getPrefix() + ".%", PropertyType.BOOLEAN.getTypeUri()).getObject(Long.class)); + + results.put("assayRunWithBooleanColumnCount", new SqlSelector(schema, """ + SELECT COUNT(DISTINCT DD.DomainURI) FROM + exp.PropertyDescriptor D\s + JOIN exp.PropertyDomain PD ON D.propertyId = PD.propertyid + JOIN exp.DomainDescriptor DD on PD.domainID = DD.domainId + WHERE D.propertyURI LIKE ? AND D.rangeURI = ?""", "urn:lsid:%:" + ExpProtocol.AssayDomainTypes.Run.getPrefix() + ".%", PropertyType.BOOLEAN.getTypeUri()).getObject(Long.class)); + + results.put("assay", assayMetrics); + } + + results.put("autoLinkedSampleSetCount", new SqlSelector(schema, "SELECT COUNT(*) FROM exp.materialsource WHERE autoLinkTargetContainer IS NOT NULL").getObject(Long.class)); + results.put("sampleSetCount", new SqlSelector(schema, "SELECT COUNT(*) FROM exp.materialsource").getObject(Long.class)); + + if (schema.getSqlDialect().isPostgreSQL()) // SQLServer does not support regular expression queries + { + Collection> numSampleCounts = new SqlSelector(schema, """ + SELECT totalCount, numberNameCount FROM + (SELECT cpastype, COUNT(*) AS totalCount from exp.material GROUP BY cpastype) t + JOIN + (SELECT cpastype, COUNT(*) AS numberNameCount FROM exp.material m WHERE m.name SIMILAR TO '[0-9.]*' GROUP BY cpastype) ns + ON t.cpastype = ns.cpastype""").getMapCollection(); + results.put("sampleSetWithNumberNamesCount", numSampleCounts.size()); + results.put("sampleSetWithOnlyNumberNamesCount", numSampleCounts.stream().filter( + map -> (Long) map.get("totalCount") > 0 && map.get("totalCount") == map.get("numberNameCount") + ).count()); + } + UserSchema userSchema = AuditLogService.getAuditLogSchema(User.getSearchUser(), ContainerManager.getRoot()); + FilteredTable table = (FilteredTable) userSchema.getTable(SampleTimelineAuditEvent.EVENT_TYPE); + + SQLFragment sql = new SQLFragment("SELECT COUNT(*)\n" + + " FROM (\n" + + " -- updates that are marked as lineage updates\n" + + " (SELECT DISTINCT transactionId\n" + + " FROM " + table.getRealTable().getFromSQL("").getSQL() +"\n" + + " WHERE islineageupdate = " + schema.getSqlDialect().getBooleanTRUE() + "\n" + + " AND comment = 'Sample was updated.'\n" + + " ) a1\n" + + " JOIN\n" + + " -- but have associated entries that are not lineage updates\n" + + " (SELECT DISTINCT transactionid\n" + + " FROM " + table.getRealTable().getFromSQL("").getSQL() + "\n" + + " WHERE islineageupdate = " + schema.getSqlDialect().getBooleanFALSE() + ") a2\n" + + " ON a1.transactionid = a2.transactionid\n" + + " )"); + + results.put("sampleLineageAuditDiscrepancyCount", new SqlSelector(schema, sql.getSQL()).getObject(Long.class)); + + results.put("sampleCount", new SqlSelector(schema, "SELECT COUNT(*) FROM exp.material").getObject(Long.class)); + results.put("aliquotCount", new SqlSelector(schema, "SELECT COUNT(*) FROM exp.material where aliquotedfromlsid IS NOT NULL").getObject(Long.class)); + results.put("sampleNullAmountCount", new SqlSelector(schema, "SELECT COUNT(*) FROM exp.material WHERE storedamount IS NULL").getObject(Long.class)); + results.put("sampleNegativeAmountCount", new SqlSelector(schema, "SELECT COUNT(*) FROM exp.material WHERE storedamount < 0").getObject(Long.class)); + results.put("sampleUnitsDifferCount", new SqlSelector(schema, "SELECT COUNT(*) from exp.material m JOIN exp.materialSource s ON m.materialsourceid = s.rowid WHERE m.units != s.metricunit").getObject(Long.class)); + results.put("sampleTypesWithoutUnitsCount", new SqlSelector(schema, "SELECT COUNT(*) from exp.materialSource WHERE category IS NULL AND metricunit IS NULL").getObject(Long.class)); + results.put("sampleTypesWithMassTypeUnit", new SqlSelector(schema, "SELECT COUNT(*) from exp.materialSource WHERE category IS NULL AND metricunit IN ('kg', 'g', 'mg', 'ug', 'ng')").getObject(Long.class)); + results.put("sampleTypesWithVolumeTypeUnit", new SqlSelector(schema, "SELECT COUNT(*) from exp.materialSource WHERE category IS NULL AND metricunit IN ('L', 'mL', 'uL')").getObject(Long.class)); + results.put("sampleTypesWithCountTypeUnit", new SqlSelector(schema, "SELECT COUNT(*) from exp.materialSource WHERE category IS NULL AND metricunit = ?", "unit").getObject(Long.class)); + + results.put("duplicateSampleMaterialNameCount", new SqlSelector(schema, "SELECT COUNT(*) as duplicateCount FROM " + + "(SELECT name, cpastype FROM exp.material WHERE cpastype <> 'Material' GROUP BY name, cpastype HAVING COUNT(*) > 1) d").getObject(Long.class)); + results.put("duplicateSpecimenMaterialNameCount", new SqlSelector(schema, "SELECT COUNT(*) as duplicateCount FROM " + + "(SELECT name, cpastype FROM exp.material WHERE cpastype = 'Material' GROUP BY name, cpastype HAVING COUNT(*) > 1) d").getObject(Long.class)); + String duplicateCaseInsensitiveSampleNameCountSql = """ + SELECT COUNT(*) FROM + ( + SELECT 1 AS found + FROM exp.material + WHERE materialsourceid IS NOT NULL + GROUP BY LOWER(name), materialsourceid + HAVING COUNT(*) > 1 + ) AS duplicates + """; + String duplicateCaseInsensitiveDataNameCountSql = """ + SELECT COUNT(*) FROM + ( + SELECT 1 AS found + FROM exp.data + WHERE classid IS NOT NULL + GROUP BY LOWER(name), classid + HAVING COUNT(*) > 1 + ) AS duplicates + """; + results.put("duplicateCaseInsensitiveSampleNameCount", new SqlSelector(schema, duplicateCaseInsensitiveSampleNameCountSql).getObject(Long.class)); + results.put("duplicateCaseInsensitiveDataNameCount", new SqlSelector(schema, duplicateCaseInsensitiveDataNameCountSql).getObject(Long.class)); + + results.put("dataClassCount", new SqlSelector(schema, "SELECT COUNT(*) FROM exp.dataclass").getObject(Long.class)); + results.put("dataClassRowCount", new SqlSelector(schema, "SELECT COUNT(*) FROM exp.data WHERE classid IN (SELECT rowid FROM exp.dataclass)").getObject(Long.class)); + results.put("dataWithDataParentsCount", new SqlSelector(schema, "SELECT COUNT(DISTINCT d.sourceApplicationId) FROM exp.data d\n" + + "JOIN exp.datainput di ON di.targetapplicationid = d.sourceapplicationid").getObject(Long.class)); + if (schema.getSqlDialect().isPostgreSQL()) + { + Collection> numDataClassObjectsCounts = new SqlSelector(schema, """ + SELECT totalCount, numberNameCount FROM + (SELECT cpastype, COUNT(*) AS totalCount from exp.data GROUP BY cpastype) t + JOIN + (SELECT cpastype, COUNT(*) AS numberNameCount FROM exp.data m WHERE m.name SIMILAR TO '[0-9.]*' GROUP BY cpastype) ns + ON t.cpastype = ns.cpastype""").getMapCollection(); + results.put("dataClassWithNumberNamesCount", numDataClassObjectsCounts.size()); + results.put("dataClassWithOnlyNumberNamesCount", numDataClassObjectsCounts.stream().filter(map -> + (Long) map.get("totalCount") > 0 && map.get("totalCount") == map.get("numberNameCount")).count()); + } + + results.put("ontologyPrincipalConceptCodeCount", new SqlSelector(schema, "SELECT COUNT(*) FROM exp.propertydescriptor WHERE principalconceptcode IS NOT NULL").getObject(Long.class)); + results.put("ontologyLookupColumnCount", new SqlSelector(schema, "SELECT COUNT(*) FROM exp.propertydescriptor WHERE concepturi = ?", OntologyService.conceptCodeConceptURI).getObject(Long.class)); + results.put("ontologyConceptSubtreeCount", new SqlSelector(schema, "SELECT COUNT(*) FROM exp.propertydescriptor WHERE conceptsubtree IS NOT NULL").getObject(Long.class)); + results.put("ontologyConceptImportColumnCount", new SqlSelector(schema, "SELECT COUNT(*) FROM exp.propertydescriptor WHERE conceptimportcolumn IS NOT NULL").getObject(Long.class)); + results.put("ontologyConceptLabelColumnCount", new SqlSelector(schema, "SELECT COUNT(*) FROM exp.propertydescriptor WHERE conceptlabelcolumn IS NOT NULL").getObject(Long.class)); + + results.put("scannableColumnCount", new SqlSelector(schema, "SELECT COUNT(*) FROM exp.propertydescriptor WHERE scannable = ?", true).getObject(Long.class)); + results.put("uniqueIdColumnCount", new SqlSelector(schema, "SELECT COUNT(*) FROM exp.propertydescriptor WHERE concepturi = ?", STORAGE_UNIQUE_ID_CONCEPT_URI).getObject(Long.class)); + results.put("sampleTypeWithUniqueIdCount", new SqlSelector(schema, """ + SELECT COUNT(DISTINCT DD.DomainURI) FROM + exp.PropertyDescriptor D\s + JOIN exp.PropertyDomain PD ON D.propertyId = PD.propertyid + JOIN exp.DomainDescriptor DD on PD.domainID = DD.domainId + WHERE D.conceptURI = ?""", STORAGE_UNIQUE_ID_CONCEPT_URI).getObject(Long.class)); + + results.put("fileColumnCount", new SqlSelector(schema, "SELECT COUNT(*) FROM exp.propertydescriptor WHERE rangeURI = ?", PropertyType.FILE_LINK.getTypeUri()).getObject(Long.class)); + results.put("sampleTypeWithFileColumnCount", new SqlSelector(schema, """ + SELECT COUNT(DISTINCT DD.DomainURI) FROM + exp.PropertyDescriptor D\s + JOIN exp.PropertyDomain PD ON D.propertyId = PD.propertyid + JOIN exp.DomainDescriptor DD on PD.domainID = DD.domainId + WHERE DD.storageSchemaName = ? AND D.rangeURI = ?""", SampleTypeDomainKind.PROVISIONED_SCHEMA_NAME, PropertyType.FILE_LINK.getTypeUri()).getObject(Long.class)); + results.put("sampleTypeWithBooleanColumnCount", new SqlSelector(schema, """ + SELECT COUNT(DISTINCT DD.DomainURI) FROM + exp.PropertyDescriptor D\s + JOIN exp.PropertyDomain PD ON D.propertyId = PD.propertyid + JOIN exp.DomainDescriptor DD on PD.domainID = DD.domainId + WHERE DD.storageSchemaName = ? AND D.rangeURI = ?""", SampleTypeDomainKind.PROVISIONED_SCHEMA_NAME, PropertyType.BOOLEAN.getTypeUri()).getObject(Long.class)); + + results.put("sampleTypeAliquotSpecificField", new SqlSelector(schema, """ + SELECT COUNT(DISTINCT D.PropertyURI) FROM + exp.PropertyDescriptor D\s + JOIN exp.PropertyDomain PD ON D.propertyId = PD.propertyid + JOIN exp.DomainDescriptor DD on PD.domainID = DD.domainId + WHERE DD.storageSchemaName = ? AND D.derivationDataScope = ?""", SampleTypeDomainKind.PROVISIONED_SCHEMA_NAME, ExpSchema.DerivationDataScopeType.ChildOnly.name()).getObject(Long.class)); + results.put("sampleTypeParentOnlyField", new SqlSelector(schema, """ + SELECT COUNT(DISTINCT D.PropertyURI) FROM + exp.PropertyDescriptor D\s + JOIN exp.PropertyDomain PD ON D.propertyId = PD.propertyid + JOIN exp.DomainDescriptor DD on PD.domainID = DD.domainId + WHERE DD.storageSchemaName = ? AND (D.derivationDataScope = ? OR D.derivationDataScope IS NULL)""", SampleTypeDomainKind.PROVISIONED_SCHEMA_NAME, ExpSchema.DerivationDataScopeType.ParentOnly.name()).getObject(Long.class)); + results.put("sampleTypeParentAndAliquotField", new SqlSelector(schema, """ + SELECT COUNT(DISTINCT D.PropertyURI) FROM + exp.PropertyDescriptor D\s + JOIN exp.PropertyDomain PD ON D.propertyId = PD.propertyid + JOIN exp.DomainDescriptor DD on PD.domainID = DD.domainId + WHERE DD.storageSchemaName = ? AND D.derivationDataScope = ?""", SampleTypeDomainKind.PROVISIONED_SCHEMA_NAME, ExpSchema.DerivationDataScopeType.All.name()).getObject(Long.class)); + + results.put("attachmentColumnCount", new SqlSelector(schema, "SELECT COUNT(*) FROM exp.propertydescriptor WHERE rangeURI = ?", PropertyType.ATTACHMENT.getTypeUri()).getObject(Long.class)); + results.put("dataClassWithAttachmentColumnCount", new SqlSelector(schema, """ + SELECT COUNT(DISTINCT DD.DomainURI) FROM + exp.PropertyDescriptor D\s + JOIN exp.PropertyDomain PD ON D.propertyId = PD.propertyid + JOIN exp.DomainDescriptor DD on PD.domainID = DD.domainId + WHERE DD.storageSchemaName = ? AND D.rangeURI = ?""", DataClassDomainKind.PROVISIONED_SCHEMA_NAME, PropertyType.ATTACHMENT.getTypeUri()).getObject(Long.class)); + results.put("dataClassWithBooleanColumnCount", new SqlSelector(schema, """ + SELECT COUNT(DISTINCT DD.DomainURI) FROM + exp.PropertyDescriptor D\s + JOIN exp.PropertyDomain PD ON D.propertyId = PD.propertyid + JOIN exp.DomainDescriptor DD on PD.domainID = DD.domainId + WHERE DD.storageSchemaName = ? AND D.rangeURI = ?""", DataClassDomainKind.PROVISIONED_SCHEMA_NAME, PropertyType.BOOLEAN.getTypeUri()).getObject(Long.class)); + + results.put("textChoiceColumnCount", new SqlSelector(schema, "SELECT COUNT(*) FROM exp.propertydescriptor WHERE concepturi = ?", TEXT_CHOICE_CONCEPT_URI).getObject(Long.class)); + results.put("multiValueTextChoiceColumnCount", new SqlSelector(schema, "SELECT COUNT(*) FROM exp.propertydescriptor WHERE rangeuri = ?", PropertyType.MULTI_CHOICE.getTypeUri()).getObject(Long.class)); + + results.put("domainsWithDateTimeColumnCount", new SqlSelector(schema, """ + SELECT COUNT(DISTINCT DD.DomainURI) FROM + exp.PropertyDescriptor D\s + JOIN exp.PropertyDomain PD ON D.propertyId = PD.propertyid + JOIN exp.DomainDescriptor DD on PD.domainID = DD.domainId + WHERE D.rangeURI = ?""", PropertyType.DATE_TIME.getTypeUri()).getObject(Long.class)); + + results.put("domainsWithDateColumnCount", new SqlSelector(schema, """ + SELECT COUNT(DISTINCT DD.DomainURI) FROM + exp.PropertyDescriptor D\s + JOIN exp.PropertyDomain PD ON D.propertyId = PD.propertyid + JOIN exp.DomainDescriptor DD on PD.domainID = DD.domainId + WHERE D.rangeURI = ?""", PropertyType.DATE.getTypeUri()).getObject(Long.class)); + + results.put("domainsWithTimeColumnCount", new SqlSelector(schema, """ + SELECT COUNT(DISTINCT DD.DomainURI) FROM + exp.PropertyDescriptor D\s + JOIN exp.PropertyDomain PD ON D.propertyId = PD.propertyid + JOIN exp.DomainDescriptor DD on PD.domainID = DD.domainId + WHERE D.rangeURI = ?""", PropertyType.TIME.getTypeUri()).getObject(Long.class)); + + results.put("maxObjectObjectId", new SqlSelector(schema, "SELECT MAX(ObjectId) FROM exp.Object").getObject(Long.class)); + results.put("maxMaterialRowId", new SqlSelector(schema, "SELECT MAX(RowId) FROM exp.Material").getObject(Long.class)); + + results.putAll(ExperimentService.get().getDomainMetrics()); + + return results; + }); + } + + ExperimentMigrationSchemaHandler handler = new ExperimentMigrationSchemaHandler(); + DatabaseMigrationService.get().registerSchemaHandler(handler); + DatabaseMigrationService.get().registerTableHandler(new MigrationTableHandler() + { + @Override + public TableInfo getTableInfo() + { + return DbSchema.get("premium", DbSchemaType.Bare).getTable("Exclusions"); + } + + @Override + public void adjustFilter(TableInfo sourceTable, SimpleFilter filter, Set containers) + { + // Include experiment runs that were copied + FilterClause includedClause = handler.getIncludedRowIdClause(sourceTable, FieldKey.fromParts("RunId")); + if (includedClause != null) + filter.addClause(includedClause); + } + }); + DatabaseMigrationService.get().registerTableHandler(new MigrationTableHandler() + { + @Override + public TableInfo getTableInfo() + { + return DbSchema.get("premium", DbSchemaType.Bare).getTable("ExclusionMaps"); + } + + @Override + public void adjustFilter(TableInfo sourceTable, SimpleFilter filter, Set containers) + { + // Include experiment runs that were copied + FilterClause includedClause = handler.getIncludedRowIdClause(sourceTable, FieldKey.fromParts("ExclusionId", "RunId")); + if (includedClause != null) + filter.addClause(includedClause); + } + }); + DatabaseMigrationService.get().registerTableHandler(new MigrationTableHandler() + { + @Override + public TableInfo getTableInfo() + { + return DbSchema.get("assayrequest", DbSchemaType.Bare).getTable("RequestRunsJunction"); + } + + @Override + public void adjustFilter(TableInfo sourceTable, SimpleFilter filter, Set containers) + { + // Include experiment runs that were copied + FilterClause includedClause = handler.getIncludedRowIdClause(sourceTable, FieldKey.fromParts("RunId")); + if (includedClause != null) + filter.addClause(includedClause); + } + }); + DatabaseMigrationService.get().registerSchemaHandler(new SampleTypeMigrationSchemaHandler()); + DataClassMigrationSchemaHandler dcHandler = new DataClassMigrationSchemaHandler(); + DatabaseMigrationService.get().registerSchemaHandler(dcHandler); + ExperimentDeleteService.setInstance(dcHandler); + } + + @Override + @NotNull + public Collection getSummary(Container c) + { + Collection list = new LinkedList<>(); + int runGroupCount = ExperimentService.get().getExperiments(c, null, false, true).size(); + if (runGroupCount > 0) + list.add(StringUtilsLabKey.pluralize(runGroupCount, "Run Group")); + + User user = HttpView.currentContext().getUser(); + + Set runTypes = ExperimentService.get().getExperimentRunTypes(c); + for (ExperimentRunType runType : runTypes) + { + if (runType == ExperimentRunType.ALL_RUNS_TYPE) + continue; + + long runCount = runType.getRunCount(user, c); + if (runCount > 0) + list.add(runCount + " runs of type " + runType.getDescription()); + } + + int dataClassCount = ExperimentService.get().getDataClasses(c, false).size(); + if (dataClassCount > 0) + list.add(dataClassCount + " Data Class" + (dataClassCount > 1 ? "es" : "")); + + int sampleTypeCount = SampleTypeService.get().getSampleTypes(c, false).size(); + if (sampleTypeCount > 0) + list.add(sampleTypeCount + " Sample Type" + (sampleTypeCount > 1 ? "s" : "")); + + return list; + } + + @Override + public @NotNull ArrayList getDetailedSummary(Container c, User user) + { + ArrayList summaries = new ArrayList<>(); + + // Assay types + long assayTypeCount = AssayService.get().getAssayProtocols(c).stream().filter(p -> p.getContainer().equals(c)).count(); + if (assayTypeCount > 0) + summaries.add(new Summary(assayTypeCount, "Assay Type")); + + // Run count + int runGroupCount = ExperimentService.get().getExperiments(c, user, false, true).size(); + if (runGroupCount > 0) + summaries.add(new Summary(runGroupCount, "Assay run")); + + // Number of Data Classes + List dataClasses = ExperimentService.get().getDataClasses(c, false); + int dataClassCount = dataClasses.size(); + if (dataClassCount > 0) + summaries.add(new Summary(dataClassCount, "Data Class")); + + ExpSchema expSchema = new ExpSchema(user, c); + + // Individual Data Class row counts + { + // The table-level container filter is set to ensure data class types are included + // that may not be defined in the target container but may have rows of data in the target container + TableInfo table = ExpSchema.TableType.DataClasses.createTable(expSchema, null, ContainerFilter.Type.CurrentPlusProjectAndShared.create(c, user)); + + // Issue 47919: The "DataCount" column is filtered to only count data in the target container + if (table instanceof ExpDataClassTableImpl tableImpl) + tableImpl.setDataCountContainerFilter(ContainerFilter.Type.Current.create(c, user)); + + Set columns = new LinkedHashSet<>(); + columns.add(ExpDataClassTable.Column.Name.name()); + columns.add(ExpDataClassTable.Column.DataCount.name()); + + Map results = new TableSelector(table, columns).getValueMap(String.class); + for (var entry : results.entrySet()) + { + long count = entry.getValue().longValue(); + if (count > 0) + summaries.add(new Summary(count, entry.getKey())); + } + } + + // Sample Types + int sampleTypeCount = SampleTypeService.get().getSampleTypes(c, false).size(); + if (sampleTypeCount > 0) + summaries.add(new Summary(sampleTypeCount, "Sample Type")); + + // Individual Sample Type row counts + { + // The table-level container filter is set to ensure data class types are included + // that may not be defined in the target container but may have rows of data in the target container + TableInfo table = ExpSchema.TableType.SampleSets.createTable(expSchema, null, ContainerFilter.Type.CurrentPlusProjectAndShared.create(c, user)); + + // Issue 51557: The "SampleCount" column is filtered to only count data in the target container + if (table instanceof ExpSampleTypeTableImpl tableImpl) + tableImpl.setSampleCountContainerFilter(ContainerFilter.Type.Current.create(c, user)); + + Set columns = new LinkedHashSet<>(); + columns.add(ExpSampleTypeTable.Column.Name.name()); + columns.add(ExpSampleTypeTable.Column.SampleCount.name()); + + Map results = new TableSelector(table, columns).getValueMap(String.class); + for (var entry : results.entrySet()) + { + long count = entry.getValue().longValue(); + if (count > 0) + { + String name = entry.getKey(); + Summary s = name.equals("MixtureBatches") + ? new Summary(count, "Batch") + : new Summary(count, name); + summaries.add(s); + } + } + } + + return summaries; + } + + @Override + public @NotNull Set> getIntegrationTests() + { + return Set.of( + DomainImpl.TestCase.class, + DomainPropertyImpl.TestCase.class, + ExpDataTableImpl.TestCase.class, + ExperimentServiceImpl.AuditDomainUriTest.class, + ExperimentServiceImpl.LineageQueryTestCase.class, + ExperimentServiceImpl.ParseInputOutputAliasTestCase.class, + ExperimentServiceImpl.TestCase.class, + ExperimentStressTest.class, + LineagePerfTest.class, + LineageTest.class, + OntologyManager.TestCase.class, + PropertyServiceImpl.TestCase.class, + SampleTypeServiceImpl.TestCase.class, + StorageNameGenerator.TestCase.class, + StorageProvisionerImpl.TestCase.class, + UniqueValueCounterTestCase.class, + XarTestPipelineJob.TestCase.class + ); + } + + @Override + public @NotNull Collection>> getIntegrationTestFactories() + { + List>> list = new ArrayList<>(super.getIntegrationTestFactories()); + list.add(new JspTestCase("/org/labkey/experiment/api/ExpDataClassDataTestCase.jsp")); + list.add(new JspTestCase("/org/labkey/experiment/api/ExpSampleTypeTestCase.jsp")); + return list; + } + + @Override + public @NotNull Set> getUnitTests() + { + return Set.of( + GraphAlgorithms.TestCase.class, + LSIDRelativizer.TestCase.class, + Lsid.TestCase.class, + LsidUtils.TestCase.class, + PropertyController.TestCase.class, + Quantity.TestCase.class, + Unit.TestCase.class + ); + } + + @Override + @NotNull + public Collection getSchemaNames() + { + return List.of( + ExpSchema.SCHEMA_NAME, + DataClassDomainKind.PROVISIONED_SCHEMA_NAME, + SampleTypeDomainKind.PROVISIONED_SCHEMA_NAME + ); + } + + @NotNull + @Override + public Collection getProvisionedSchemaNames() + { + return PageFlowUtil.set(DataClassDomainKind.PROVISIONED_SCHEMA_NAME, SampleTypeDomainKind.PROVISIONED_SCHEMA_NAME); + } +} diff --git a/list/src/org/labkey/list/model/ListDomainKind.java b/list/src/org/labkey/list/model/ListDomainKind.java index f29b9169164..8ae2881d322 100644 --- a/list/src/org/labkey/list/model/ListDomainKind.java +++ b/list/src/org/labkey/list/model/ListDomainKind.java @@ -1,750 +1,750 @@ -/* - * Copyright (c) 2013-2018 LabKey Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.labkey.list.model; - -import org.apache.commons.lang3.EnumUtils; -import org.apache.commons.lang3.StringUtils; -import org.jetbrains.annotations.NotNull; -import org.jetbrains.annotations.Nullable; -import org.labkey.api.action.ApiUsageException; -import org.labkey.api.attachments.AttachmentService; -import org.labkey.api.collections.CaseInsensitiveHashMap; -import org.labkey.api.collections.CaseInsensitiveHashSet; -import org.labkey.api.compliance.ComplianceService; -import org.labkey.api.data.ColumnInfo; -import org.labkey.api.data.Container; -import org.labkey.api.data.ContainerFilter; -import org.labkey.api.data.DbScope; -import org.labkey.api.data.JdbcType; -import org.labkey.api.data.PropertyStorageSpec; -import org.labkey.api.data.RuntimeSQLException; -import org.labkey.api.data.SQLFragment; -import org.labkey.api.data.SimpleFilter; -import org.labkey.api.data.TableInfo; -import org.labkey.api.data.TableSelector; -import org.labkey.api.defaults.DefaultValueService; -import org.labkey.api.di.DataIntegrationService; -import org.labkey.api.exp.DomainNotFoundException; -import org.labkey.api.exp.Lsid; -import org.labkey.api.exp.LsidManager; -import org.labkey.api.exp.OntologyManager; -import org.labkey.api.exp.PropertyDescriptor; -import org.labkey.api.exp.PropertyType; -import org.labkey.api.exp.TemplateInfo; -import org.labkey.api.exp.api.ExperimentService; -import org.labkey.api.exp.list.ListDefinition; -import org.labkey.api.exp.list.ListDefinition.KeyType; -import org.labkey.api.exp.list.ListService; -import org.labkey.api.exp.list.ListUrls; -import org.labkey.api.exp.property.AbstractDomainKind; -import org.labkey.api.exp.property.Domain; -import org.labkey.api.exp.property.DomainProperty; -import org.labkey.api.exp.property.DomainUtil; -import org.labkey.api.exp.property.PropertyService; -import org.labkey.api.gwt.client.model.GWTDomain; -import org.labkey.api.gwt.client.model.GWTIndex; -import org.labkey.api.gwt.client.model.GWTPropertyDescriptor; -import org.labkey.api.lists.permissions.DesignListPermission; -import org.labkey.api.query.FieldKey; -import org.labkey.api.query.MetadataUnavailableException; -import org.labkey.api.query.QueryService; -import org.labkey.api.query.ValidationException; -import org.labkey.api.security.User; -import org.labkey.api.settings.AppProps; -import org.labkey.api.util.PageFlowUtil; -import org.labkey.api.util.Pair; -import org.labkey.api.view.ActionURL; -import org.labkey.api.view.NotFoundException; -import org.labkey.api.writer.ContainerUser; -import org.labkey.data.xml.domainTemplate.DomainTemplateType; -import org.labkey.data.xml.domainTemplate.ListOptionsType; -import org.labkey.data.xml.domainTemplate.ListTemplateType; -import org.labkey.list.view.ListItemAttachmentParent; -import org.springframework.dao.DataIntegrityViolationException; - -import java.util.ArrayList; -import java.util.Collection; -import java.util.Collections; -import java.util.HashMap; -import java.util.HashSet; -import java.util.List; -import java.util.Map; -import java.util.Set; -import java.util.stream.Collectors; - -import static org.labkey.api.exp.property.DomainTemplate.findProperty; - -public abstract class ListDomainKind extends AbstractDomainKind -{ - /* - * the columns common to all lists - */ - private final static Set BASE_PROPERTIES; - private ListDefinitionImpl _list; - private final static int MAX_NAME_LENGTH = 200; - private static final Set RESERVED_NAMES; - - static - { - BASE_PROPERTIES = PageFlowUtil.set(new PropertyStorageSpec("entityId", JdbcType.GUID).setNullable(false), - new PropertyStorageSpec("created", JdbcType.TIMESTAMP), - new PropertyStorageSpec("createdBy", JdbcType.INTEGER), - new PropertyStorageSpec("modified", JdbcType.TIMESTAMP), - new PropertyStorageSpec("modifiedBy", JdbcType.INTEGER), - new PropertyStorageSpec("lastIndexed", JdbcType.TIMESTAMP), - new PropertyStorageSpec("container", JdbcType.GUID).setNullable(false), - new PropertyStorageSpec(DataIntegrationService.Columns.TransformImportHash.getColumnName(), JdbcType.VARCHAR, 256)); - RESERVED_NAMES = DomainUtil.getNamesAndLabels( - BASE_PROPERTIES.stream().map(PropertyStorageSpec::getName).collect(Collectors.toSet())); - } - - public void setListDefinition(ListDefinitionImpl list) - { - _list = list; - } - - @Override - public String getTypeLabel(Domain domain) - { - return "List '" + domain.getName() + "'"; - } - - @Override - public SQLFragment sqlObjectIdsInDomain(Domain domain) - { - throw new UnsupportedOperationException("sqlObjectIdsInDomain NYI for ListDomainKind"); - } - - @Override - public ActionURL urlShowData(Domain domain, ContainerUser containerUser) - { - ListDefinition listDef = ListService.get().getList(domain); - if (null == listDef) - return null; - return listDef.urlShowData(); - } - - @Nullable - @Override - public ActionURL urlEditDefinition(Domain domain, ContainerUser containerUser) - { - ListDefinition listDef = ListService.get().getList(domain); - if (null == listDef) - return null; - return listDef.urlShowDefinition(); - } - - @Override - public boolean allowAttachmentProperties() - { - return true; - } - - @Override - public boolean allowMultiChoiceProperties() - { - return true; - } - - @Override - public boolean showDefaultValueSettings() - { - return true; - } - - @Override - public boolean allowUniqueConstraintProperties() - { - return true; - } - - @Override - public boolean allowCalculatedFields() - { - return true; - } - - @Override - public Priority getPriority(String domainURI) - { - Lsid lsid = new Lsid(domainURI); - return getKindName().equals(lsid.getNamespacePrefix()) ? Priority.MEDIUM : null; - } - - - @Override - public Set getBaseProperties(Domain domain) - { - Set specs = new HashSet<>(BASE_PROPERTIES); - specs.addAll(super.getBaseProperties(domain)); - return specs; - } - - @Override - public Set getMandatoryPropertyNames(Domain domain) - { - Set properties = super.getMandatoryPropertyNames(domain); - properties.addAll(getAdditionalProtectedPropertyNames(domain)); - return properties; - } - - /** - * Returns the List's primary key as a field to get special treatment elsewhere, despite being property driven. - */ - @Override - public Set getAdditionalProtectedProperties(Domain domain) - { - Set specs = new HashSet<>(super.getAdditionalProtectedProperties(domain)); - - ListDefinition listDef = ListService.get().getList(domain); - if (null != listDef) - { - String keyName = listDef.getKeyName(); - JdbcType keyType = listDef.getKeyType() == KeyType.Varchar ? JdbcType.VARCHAR : JdbcType.INTEGER; - int keySize = keyType == JdbcType.VARCHAR ? 4000 : 0; - specs.add(new PropertyStorageSpec(keyName, keyType, keySize, PropertyStorageSpec.Special.PrimaryKey)); - } - - return specs; - } - - @Override - public PropertyStorageSpec getPropertySpec(PropertyDescriptor pd, Domain domain) - { - ListDefinition list = ListService.get().getList(domain); - if (null == list) - list = _list; - - if (null != list) - { - if (pd.getName().equalsIgnoreCase(list.getKeyName())) - { - PropertyStorageSpec key = getKeyProperty(list, pd.getStorageColumnName()); - assert key.isPrimaryKey(); - _list = null; - return key; - } - } - return super.getPropertySpec(pd, domain); - } - - abstract PropertyStorageSpec getKeyProperty(ListDefinition list, String storageColumnName); - - abstract KeyType getDefaultKeyType(); - - abstract Collection getSupportedKeyTypes(); - - @Override - public @NotNull Set getReservedPropertyNames(Domain domain, User user) - { - return RESERVED_NAMES; - } - - @Override - public String getStorageSchemaName() - { - return ListSchema.getInstance().getSchemaName(); - } - - @Override - public DbScope getScope() - { - return ListSchema.getInstance().getSchema().getScope(); - } - - @Override - public Set getPropertyIndices(Domain domain) - { - // return PageFlowUtil.set(new PropertyStorageSpec.Index(false, ListDomainKind.KEY_FIELD)); - return Collections.emptySet(); // TODO: Allow this to return the Key Column - } - - public static Lsid generateDomainURI(Container c, KeyType keyType, @Nullable ListDefinition.Category category) - { - String type = getType(keyType, category); - Lsid lsid; - // assure LSID does not collide with previous lsids that may have had number names - do - { - String dbSeqStr = String.valueOf(LsidManager.getLsidPrefixDbSeq(type, 1).next()); - lsid = new Lsid(type, "Folder-" + c.getRowId(), dbSeqStr); - } while (OntologyManager.getDomainDescriptor(lsid.toString(), c) != null); - - return lsid; - } - - public static Lsid createPropertyURI(String listName, String columnName, Container c, ListDefinition.KeyType keyType) - { - StringBuilder typeURI = getBaseURI(listName, getType(keyType, null), c); - typeURI.append(".").append(PageFlowUtil.encode(columnName)); - return new Lsid(typeURI.toString()); - } - - private static String getType(KeyType keyType, @Nullable ListDefinition.Category category) - { - String type; - switch (keyType) - { - case Integer: - case AutoIncrementInteger: - if (category != null) - type = PicklistDomainKind.NAMESPACE_PREFIX; - else - type = IntegerListDomainKind.NAMESPACE_PREFIX; - break; - case Varchar: - type = VarcharListDomainKind.NAMESPACE_PREFIX; - break; - default: - throw new IllegalStateException(); - } - return type; - } - - private static StringBuilder getBaseURI(String listName, String type, Container c) - { - return new StringBuilder("urn:lsid:") - .append(PageFlowUtil.encode(AppProps.getInstance().getDefaultLsidAuthority())) - .append(":").append(type).append(".Folder-").append(c.getRowId()).append(":") - .append(PageFlowUtil.encode(listName)); - } - - @Override - public TableInfo getTableInfo(User user, Container container, String name, @Nullable ContainerFilter cf) - { - return new ListQuerySchema(user, container).createTable(name, cf); - } - - @Override - public boolean isDeleteAllDataOnFieldImport() - { - return true; - } - - @Override - public boolean canCreateDefinition(User user, Container container) - { - return container.hasPermission("ListDomainKind.canCreateDefinition", user, DesignListPermission.class); - } - - @Override - public boolean canEditDefinition(User user, Domain domain) - { - Container c = domain.getContainer(); - return c.hasPermission("ListDomainKind.canEditDefinition", user, DesignListPermission.class); - } - - @Override - public ActionURL urlCreateDefinition(String schemaName, String queryName, Container container, User user) - { - return PageFlowUtil.urlProvider(ListUrls.class).getCreateListURL(container); - } - - @Override - public Class getTypeClass() - { - return ListDomainKindProperties.class; - } - - @Override - public void validateDomainName(Container container, User user, @Nullable Domain domain, String name) - { - if (StringUtils.isEmpty(name)) - throw new ApiUsageException("List name is required."); - if (name.length() > MAX_NAME_LENGTH) - throw new ApiUsageException("List name cannot be longer than " + MAX_NAME_LENGTH + " characters."); - if (ListService.get().getList(container, name, domain == null) != null) - throw new ApiUsageException("The name '" + name + "' is already in use."); - } - - @Override - public Domain createDomain(GWTDomain domain, ListDomainKindProperties listProperties, Container container, User user, @Nullable TemplateInfo templateInfo, boolean forUpdate) - { - String name = StringUtils.trimToEmpty(domain.getName()); - validateDomainName(container, user, null, name); - - String keyName = listProperties.getKeyName(); - if (StringUtils.isEmpty(keyName)) - throw new ApiUsageException("List keyName is required"); - - KeyType keyType = getDefaultKeyType(); - - if (null != listProperties.getKeyType()) - { - String rawKeyType = listProperties.getKeyType(); - if (EnumUtils.isValidEnum(KeyType.class, rawKeyType)) - keyType = KeyType.valueOf(rawKeyType); - else - throw new ApiUsageException("List keyType provided does not exist."); - } - - ListDefinition.Category category; - try - { - category = listProperties.getCategory() != null ? ListDefinition.Category.valueOf(listProperties.getCategory()) : null; - } - catch (IllegalArgumentException e) - { - throw new ApiUsageException(String.format("List category type provided (%s) does not exist.", listProperties.getCategory())); - } - - if (!getSupportedKeyTypes().contains(keyType)) - throw new ApiUsageException("List keyType provided is not supported for list domain kind (" + getKindName() + ")."); - - ListDefinition list = ListService.get().createList(container, name, keyType, templateInfo, category); - list.setKeyName(keyName); - - // Issue 45042: Allow for the list description to be set via the create domain API calls - if (listProperties.getDescription() == null && domain.getDescription() != null) - listProperties.setDescription(domain.getDescription()); - list.setDescription(listProperties.getDescription()); - - List properties = (List)domain.getFields(); - List indices = (List)domain.getIndices(); - - try (DbScope.Transaction tx = ExperimentService.get().ensureTransaction()) - { - Domain d = list.getDomain(forUpdate); - - Set reservedNames = getReservedPropertyNames(d, user); - Set lowerReservedNames = reservedNames.stream().map(String::toLowerCase).collect(Collectors.toSet()); - - Map defaultValues = new HashMap<>(); - Set propertyUris = new CaseInsensitiveHashSet(); - for (GWTPropertyDescriptor pd : properties) - { - String propertyName = pd.getName().toLowerCase(); - if (lowerReservedNames.contains(propertyName)) - { - if (pd.getLabel() == null) - pd.setLabel(pd.getName()); - pd.setName("_" + pd.getName()); - } - - DomainUtil.addProperty(d, pd, defaultValues, propertyUris, null); - } - - d.setPropertyIndices(indices, lowerReservedNames); - - // TODO: This looks like the wrong order to me -- we should updateListProperties() (persist the indexing - // settings and handle potential transitions) before calling save() (which indexes the list). Since this is - // the create case there's no data to index, but there is meta data... - list.save(user, true, listProperties.getAuditRecordMap(), domain.getCalculatedFields()); - updateListProperties(container, user, list.getListId(), listProperties); - - QueryService.get().saveCalculatedFieldsMetadata(ListQuerySchema.NAME, name, null, domain.getCalculatedFields(), false, user, container); - - DefaultValueService.get().setDefaultValues(container, defaultValues); - - tx.commit(); - } - catch (Exception e) - { - throw new RuntimeException(e); - } - - return list.getDomain(forUpdate); - } - - @Override - public @NotNull ValidationException updateDomain(GWTDomain original, GWTDomain update, - ListDomainKindProperties listProperties, Container container, User user, boolean includeWarnings, @Nullable String auditUserComment) - { - ValidationException exception; - - try (DbScope.Transaction transaction = ListManager.get().getListMetadataSchema().getScope().ensureTransaction()) - { - exception = new ValidationException(); - StringBuilder changeDetails = new StringBuilder(); - - Domain domain = PropertyService.get().getDomain(container, original.getDomainURI()); - if (null == domain) - return exception.addGlobalError("Expected domain for list: " + original.getName()); - - ListDefinition listDefinition = ListService.get().getList(domain); - TableInfo table = listDefinition.getTable(user); - - if (null == table) - return exception.addGlobalError("Expected table for list: " + listDefinition.getName()); - - // Handle cases when existing key field is null or is not provided in the updated domainDesign - GWTPropertyDescriptor key = findField(listDefinition.getKeyName(), original.getFields()); - if (null != key) - { - int id = key.getPropertyId(); - GWTPropertyDescriptor newKey = findField(id, update.getFields()); - if (null == newKey) - { - return exception.addGlobalError("Key field not provided, expecting key field '" + key.getName() + "'"); - } - else if (!key.getName().equalsIgnoreCase(newKey.getName())) - { - return exception.addGlobalError("Cannot change key field name"); - } - } - else - { - return exception.addGlobalError("Key field not found for list '" + listDefinition.getName() + "'"); - } - - //handle name change - String updatedName = StringUtils.trim(update.getName()); - boolean hasNameChange = !original.getName().equals(updatedName); - if (hasNameChange) - { - validateDomainName(container, user, domain, updatedName); - changeDetails.append("The name of the list domain '" + original.getName() + "' was changed to '" + updatedName + "'."); - } - - //return if there are errors before moving forward with the save - if (exception.hasErrors()) - { - return exception; - } - - //update list properties - Map oldProps = null; - Map newProps = null; - if (null != listProperties) - { - if (listProperties.getDomainId() != original.getDomainId() || listProperties.getDomainId() != update.getDomainId()) - return exception.addGlobalError("domainId for the list does not match old or the new domain"); - if (!original.getDomainURI().equals(update.getDomainURI())) - return exception.addGlobalError("domainURI mismatch between old and new domain"); - - Pair, Map> updatedProps = updateListProperties(container, user, listDefinition.getListId(), listProperties); - oldProps = updatedProps.first; - newProps = updatedProps.second; - } - // Issue 45042: Allow for the list description to be set via the save domain API calls - else if (update.getDescription() != null) - { - listProperties = getListProperties(container, user, listDefinition.getListId()); - listProperties.setDescription(update.getDescription()); - Pair, Map> updatedProps = updateListProperties(container, user, listDefinition.getListId(), listProperties); - oldProps = updatedProps.first; - newProps = updatedProps.second; - } - - //update domain design properties - try - { - //handle attachment cols - Map modifiedAttachmentColumns = new CaseInsensitiveHashMap<>(); - - for (DomainProperty oldProp : domain.getProperties()) - { - if (PropertyType.ATTACHMENT.equals(oldProp.getPropertyDescriptor().getPropertyType())) - { - GWTPropertyDescriptor newGWTProp = findField(oldProp.getPropertyId(), update.getFields()); - if (null == newGWTProp || !PropertyType.ATTACHMENT.equals(PropertyType.getFromURI(newGWTProp.getConceptURI(), newGWTProp.getRangeURI(), null))) - { - ColumnInfo column = table.getColumn(oldProp.getPropertyDescriptor().getName()); - if (null != column) - modifiedAttachmentColumns.put(oldProp.getPropertyDescriptor().getName(), column); - } - } - } - - Collection> attachmentMapCollection = null; - if (!modifiedAttachmentColumns.isEmpty()) - { - List columns = new ArrayList<>(modifiedAttachmentColumns.values()); - columns.add(table.getColumn("entityId")); - attachmentMapCollection = new TableSelector(table, columns, null, null).getMapCollection(); - } - - // Remove attachments from any attachment columns that are removed or no longer attachment columns - if (null != attachmentMapCollection) - { - for (Map map : attachmentMapCollection) - { - String entityId = (String) map.get("entityId"); - ListItemAttachmentParent parent = new ListItemAttachmentParent(entityId, container); - for (Map.Entry entry : map.entrySet()) - if (null != entry.getValue() && modifiedAttachmentColumns.containsKey(entry.getKey())) - AttachmentService.get().deleteAttachment(parent, entry.getValue().toString(), user); - } - } - - //update domain properties - exception.addErrors(DomainUtil.updateDomainDescriptor(original, update, container, user, hasNameChange, changeDetails.toString(), auditUserComment, oldProps, newProps)); - - QueryService.get().saveCalculatedFieldsMetadata(ListQuerySchema.NAME, update.getQueryName(), hasNameChange ? update.getName() : null, update.getCalculatedFields(), !original.getCalculatedFields().isEmpty(), user, container); - } - catch (RuntimeSQLException x) - { - // issue 19202 - check for null value exceptions in case provided file data not contain the column - // and return a better error message - String message = x.getMessage(); - if (x.isNullValueException()) - { - message = "The provided data does not contain the specified '" + listDefinition.getKeyName() + "' field or contains null key values."; - } - return exception.addGlobalError(message); - } - catch (DataIntegrityViolationException | MetadataUnavailableException x) - { - return exception.addGlobalError("A data error occurred: " + x.getMessage()); - } - - if (!exception.hasErrors()) - { - transaction.commit(); - } - return exception; - } - } - - private ListDomainKindProperties getListProperties(Container container, User user, int listId) - { - SimpleFilter filter = SimpleFilter.createContainerFilter(container); - filter.addCondition(FieldKey.fromParts("ListId"), listId); - return new TableSelector(ListManager.get().getListMetadataTable(), filter, null).getObject(ListDomainKindProperties.class); - } - - private Pair, Map> updateListProperties(Container container, User user, int listId, ListDomainKindProperties listProperties) - { - ListDomainKindProperties existingListProps = getListProperties(container, user, listId); - - Map oldProps = existingListProps == null ? null : existingListProps.getAuditRecordMap(); - Map newProps = listProperties == null ? oldProps : listProperties.getAuditRecordMap(); - //merge existing and new properties - ListDomainKindProperties updatedListProps = updateListProperties(existingListProps, listProperties); - - ListManager.get().update(user, container, updatedListProps); - - return new Pair<>(oldProps, newProps); - } - - //updates list properties except listId, domainId, keyName, keyType, and lastIndexed - private ListDomainKindProperties updateListProperties(ListDomainKindProperties existingListProps, ListDomainKindProperties newListProps) - { - - ListDomainKindProperties updatedListProps = new ListDomainKindProperties(existingListProps); - if (null != newListProps.getName()) - updatedListProps.setName(newListProps.getName().trim()); - - updatedListProps.setTitleColumn(newListProps.getTitleColumn()); - updatedListProps.setDescription(newListProps.getDescription()); - updatedListProps.setAllowDelete(newListProps.isAllowDelete()); - updatedListProps.setAllowUpload(newListProps.isAllowUpload()); - updatedListProps.setAllowExport(newListProps.isAllowExport()); - updatedListProps.setCategory(newListProps.getCategory()); - updatedListProps.setEntireListTitleTemplate(newListProps.getEntireListTitleTemplate()); - updatedListProps.setEntireListIndexSetting(newListProps.getEntireListIndexSetting()); - updatedListProps.setEntireListBodySetting(newListProps.getEntireListBodySetting()); - updatedListProps.setEachItemTitleTemplate(newListProps.getEachItemTitleTemplate()); - updatedListProps.setEachItemBodySetting(newListProps.getEachItemBodySetting()); - updatedListProps.setEntireListIndex(newListProps.isEntireListIndex()); - updatedListProps.setEntireListBodyTemplate(newListProps.getEntireListBodyTemplate()); - updatedListProps.setEachItemIndex(newListProps.isEachItemIndex()); - updatedListProps.setEachItemBodyTemplate(newListProps.getEachItemBodyTemplate()); - updatedListProps.setFileAttachmentIndex(newListProps.isFileAttachmentIndex()); - - return updatedListProps; - } - - private GWTPropertyDescriptor findField(String name, List fields) - { - for (GWTPropertyDescriptor f : fields) - { - if (name.equalsIgnoreCase(f.getName())) - return f; - } - return null; - } - - private GWTPropertyDescriptor findField(int id, List fields) - { - if (id > 0) - { - for (GWTPropertyDescriptor f : fields) - { - if (id == f.getPropertyId()) - return f; - } - } - return null; - } - - @Override - public @Nullable ListDomainKindProperties getDomainKindProperties(GWTDomain domain, Container container, User user) - { - ListDefinition list = domain != null ? ListService.get().getList(PropertyService.get().getDomain(domain.getDomainId())) : null; - return ListManager.get().getListDomainKindProperties(container, list != null ? list.getListId() : null); - } - - @Override - public void deleteDomain(User user, Domain domain, String userComment) - { - ListDefinition list = ListService.get().getList(domain); - if (list == null) - throw new NotFoundException("List not found: " + domain.getTypeURI()); - - try - { - list.delete(user, userComment); - } - catch (DomainNotFoundException e) - { - throw new NotFoundException(e.getMessage()); - } - } - - @Override - public void invalidate(Domain domain) - { - super.invalidate(domain); - - ListDefinition list = ListService.get().getList(domain); - if (list != null) - ListManager.get().indexList(list); - } - - @Override - public boolean matchesTemplateXML(String templateName, DomainTemplateType template, List properties) - { - if (!(template instanceof ListTemplateType)) - return false; - - ListOptionsType options = ((ListTemplateType) template).getOptions(); - if (options == null) - throw new IllegalArgumentException("List template requires specifying a keyCol"); - - String keyName = options.getKeyCol(); - if (keyName == null) - throw new IllegalArgumentException("List template requires specifying a keyCol"); - - Pair pair = findProperty(templateName, properties, keyName); - GWTPropertyDescriptor prop = pair.first; - - PropertyType type = PropertyType.getFromURI(prop.getConceptURI(), prop.getRangeURI()); - - return type.equals(getDefaultKeyType().getPropertyType()); - } - - public void ensureBaseProperties(Domain d) - { - var props = getBaseProperties(d); - } - - @Override - public boolean supportsPhiLevel() - { - return ComplianceService.get().isComplianceSupported(); - } -} +/* + * Copyright (c) 2013-2018 LabKey Corporation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.labkey.list.model; + +import org.apache.commons.lang3.EnumUtils; +import org.apache.commons.lang3.StringUtils; +import org.jetbrains.annotations.NotNull; +import org.jetbrains.annotations.Nullable; +import org.labkey.api.action.ApiUsageException; +import org.labkey.api.attachments.AttachmentService; +import org.labkey.api.collections.CaseInsensitiveHashMap; +import org.labkey.api.collections.CaseInsensitiveHashSet; +import org.labkey.api.compliance.ComplianceService; +import org.labkey.api.data.ColumnInfo; +import org.labkey.api.data.Container; +import org.labkey.api.data.ContainerFilter; +import org.labkey.api.data.DbScope; +import org.labkey.api.data.JdbcType; +import org.labkey.api.data.PropertyStorageSpec; +import org.labkey.api.data.RuntimeSQLException; +import org.labkey.api.data.SQLFragment; +import org.labkey.api.data.SimpleFilter; +import org.labkey.api.data.TableInfo; +import org.labkey.api.data.TableSelector; +import org.labkey.api.defaults.DefaultValueService; +import org.labkey.api.di.DataIntegrationService; +import org.labkey.api.exp.DomainNotFoundException; +import org.labkey.api.exp.Lsid; +import org.labkey.api.exp.LsidManager; +import org.labkey.api.exp.OntologyManager; +import org.labkey.api.exp.PropertyDescriptor; +import org.labkey.api.exp.PropertyType; +import org.labkey.api.exp.TemplateInfo; +import org.labkey.api.exp.api.ExperimentService; +import org.labkey.api.exp.list.ListDefinition; +import org.labkey.api.exp.list.ListDefinition.KeyType; +import org.labkey.api.exp.list.ListService; +import org.labkey.api.exp.list.ListUrls; +import org.labkey.api.exp.property.AbstractDomainKind; +import org.labkey.api.exp.property.Domain; +import org.labkey.api.exp.property.DomainProperty; +import org.labkey.api.exp.property.DomainUtil; +import org.labkey.api.exp.property.PropertyService; +import org.labkey.api.gwt.client.model.GWTDomain; +import org.labkey.api.gwt.client.model.GWTIndex; +import org.labkey.api.gwt.client.model.GWTPropertyDescriptor; +import org.labkey.api.lists.permissions.DesignListPermission; +import org.labkey.api.query.FieldKey; +import org.labkey.api.query.MetadataUnavailableException; +import org.labkey.api.query.QueryService; +import org.labkey.api.query.ValidationException; +import org.labkey.api.security.User; +import org.labkey.api.settings.AppProps; +import org.labkey.api.util.PageFlowUtil; +import org.labkey.api.util.Pair; +import org.labkey.api.view.ActionURL; +import org.labkey.api.view.NotFoundException; +import org.labkey.api.writer.ContainerUser; +import org.labkey.data.xml.domainTemplate.DomainTemplateType; +import org.labkey.data.xml.domainTemplate.ListOptionsType; +import org.labkey.data.xml.domainTemplate.ListTemplateType; +import org.labkey.list.view.ListItemAttachmentParent; +import org.springframework.dao.DataIntegrityViolationException; + +import java.util.ArrayList; +import java.util.Collection; +import java.util.Collections; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.stream.Collectors; + +import static org.labkey.api.exp.property.DomainTemplate.findProperty; + +public abstract class ListDomainKind extends AbstractDomainKind +{ + /* + * the columns common to all lists + */ + private final static Set BASE_PROPERTIES; + private ListDefinitionImpl _list; + private final static int MAX_NAME_LENGTH = 200; + private static final Set RESERVED_NAMES; + + static + { + BASE_PROPERTIES = PageFlowUtil.set(new PropertyStorageSpec("entityId", JdbcType.GUID).setNullable(false), + new PropertyStorageSpec("created", JdbcType.TIMESTAMP), + new PropertyStorageSpec("createdBy", JdbcType.INTEGER), + new PropertyStorageSpec("modified", JdbcType.TIMESTAMP), + new PropertyStorageSpec("modifiedBy", JdbcType.INTEGER), + new PropertyStorageSpec("lastIndexed", JdbcType.TIMESTAMP), + new PropertyStorageSpec("container", JdbcType.GUID).setNullable(false), + new PropertyStorageSpec(DataIntegrationService.Columns.TransformImportHash.getColumnName(), JdbcType.VARCHAR, 256)); + RESERVED_NAMES = DomainUtil.getNamesAndLabels( + BASE_PROPERTIES.stream().map(PropertyStorageSpec::getName).collect(Collectors.toSet())); + } + + public void setListDefinition(ListDefinitionImpl list) + { + _list = list; + } + + @Override + public String getTypeLabel(Domain domain) + { + return "List '" + domain.getName() + "'"; + } + + @Override + public SQLFragment sqlObjectIdsInDomain(Domain domain) + { + throw new UnsupportedOperationException("sqlObjectIdsInDomain NYI for ListDomainKind"); + } + + @Override + public ActionURL urlShowData(Domain domain, ContainerUser containerUser) + { + ListDefinition listDef = ListService.get().getList(domain); + if (null == listDef) + return null; + return listDef.urlShowData(); + } + + @Nullable + @Override + public ActionURL urlEditDefinition(Domain domain, ContainerUser containerUser) + { + ListDefinition listDef = ListService.get().getList(domain); + if (null == listDef) + return null; + return listDef.urlShowDefinition(); + } + + @Override + public boolean allowAttachmentProperties() + { + return true; + } + + @Override + public boolean allowMultiChoiceProperties() + { + return true; + } + + @Override + public boolean showDefaultValueSettings() + { + return true; + } + + @Override + public boolean allowUniqueConstraintProperties() + { + return true; + } + + @Override + public boolean allowCalculatedFields() + { + return true; + } + + @Override + public Priority getPriority(String domainURI) + { + Lsid lsid = new Lsid(domainURI); + return getKindName().equals(lsid.getNamespacePrefix()) ? Priority.MEDIUM : null; + } + + + @Override + public Set getBaseProperties(Domain domain) + { + Set specs = new HashSet<>(BASE_PROPERTIES); + specs.addAll(super.getBaseProperties(domain)); + return specs; + } + + @Override + public Set getMandatoryPropertyNames(Domain domain) + { + Set properties = super.getMandatoryPropertyNames(domain); + properties.addAll(getAdditionalProtectedPropertyNames(domain)); + return properties; + } + + /** + * Returns the List's primary key as a field to get special treatment elsewhere, despite being property driven. + */ + @Override + public Set getAdditionalProtectedProperties(Domain domain) + { + Set specs = new HashSet<>(super.getAdditionalProtectedProperties(domain)); + + ListDefinition listDef = ListService.get().getList(domain); + if (null != listDef) + { + String keyName = listDef.getKeyName(); + JdbcType keyType = listDef.getKeyType() == KeyType.Varchar ? JdbcType.VARCHAR : JdbcType.INTEGER; + int keySize = keyType == JdbcType.VARCHAR ? 4000 : 0; + specs.add(new PropertyStorageSpec(keyName, keyType, keySize, PropertyStorageSpec.Special.PrimaryKey)); + } + + return specs; + } + + @Override + public PropertyStorageSpec getPropertySpec(PropertyDescriptor pd, Domain domain) + { + ListDefinition list = ListService.get().getList(domain); + if (null == list) + list = _list; + + if (null != list) + { + if (pd.getName().equalsIgnoreCase(list.getKeyName())) + { + PropertyStorageSpec key = getKeyProperty(list, pd.getStorageColumnName()); + assert key.isPrimaryKey(); + _list = null; + return key; + } + } + return super.getPropertySpec(pd, domain); + } + + abstract PropertyStorageSpec getKeyProperty(ListDefinition list, String storageColumnName); + + abstract KeyType getDefaultKeyType(); + + abstract Collection getSupportedKeyTypes(); + + @Override + public @NotNull Set getReservedPropertyNames(Domain domain, User user) + { + return RESERVED_NAMES; + } + + @Override + public String getStorageSchemaName() + { + return ListSchema.getInstance().getSchemaName(); + } + + @Override + public DbScope getScope() + { + return ListSchema.getInstance().getSchema().getScope(); + } + + @Override + public Set getPropertyIndices(Domain domain) + { + // return PageFlowUtil.set(new PropertyStorageSpec.Index(false, ListDomainKind.KEY_FIELD)); + return Collections.emptySet(); // TODO: Allow this to return the Key Column + } + + public static Lsid generateDomainURI(Container c, KeyType keyType, @Nullable ListDefinition.Category category) + { + String type = getType(keyType, category); + Lsid lsid; + // assure LSID does not collide with previous lsids that may have had number names + do + { + String dbSeqStr = String.valueOf(LsidManager.getLsidPrefixDbSeq(type, 1).next()); + lsid = new Lsid(type, "Folder-" + c.getRowId(), dbSeqStr); + } while (OntologyManager.getDomainDescriptor(lsid.toString(), c) != null); + + return lsid; + } + + public static Lsid createPropertyURI(String listName, String columnName, Container c, ListDefinition.KeyType keyType) + { + StringBuilder typeURI = getBaseURI(listName, getType(keyType, null), c); + typeURI.append(".").append(PageFlowUtil.encode(columnName)); + return new Lsid(typeURI.toString()); + } + + private static String getType(KeyType keyType, @Nullable ListDefinition.Category category) + { + String type; + switch (keyType) + { + case Integer: + case AutoIncrementInteger: + if (category != null) + type = PicklistDomainKind.NAMESPACE_PREFIX; + else + type = IntegerListDomainKind.NAMESPACE_PREFIX; + break; + case Varchar: + type = VarcharListDomainKind.NAMESPACE_PREFIX; + break; + default: + throw new IllegalStateException(); + } + return type; + } + + private static StringBuilder getBaseURI(String listName, String type, Container c) + { + return new StringBuilder("urn:lsid:") + .append(PageFlowUtil.encode(AppProps.getInstance().getDefaultLsidAuthority())) + .append(":").append(type).append(".Folder-").append(c.getRowId()).append(":") + .append(PageFlowUtil.encode(listName)); + } + + @Override + public TableInfo getTableInfo(User user, Container container, String name, @Nullable ContainerFilter cf) + { + return new ListQuerySchema(user, container).createTable(name, cf); + } + + @Override + public boolean isDeleteAllDataOnFieldImport() + { + return true; + } + + @Override + public boolean canCreateDefinition(User user, Container container) + { + return container.hasPermission("ListDomainKind.canCreateDefinition", user, DesignListPermission.class); + } + + @Override + public boolean canEditDefinition(User user, Domain domain) + { + Container c = domain.getContainer(); + return c.hasPermission("ListDomainKind.canEditDefinition", user, DesignListPermission.class); + } + + @Override + public ActionURL urlCreateDefinition(String schemaName, String queryName, Container container, User user) + { + return PageFlowUtil.urlProvider(ListUrls.class).getCreateListURL(container); + } + + @Override + public Class getTypeClass() + { + return ListDomainKindProperties.class; + } + + @Override + public void validateDomainName(Container container, User user, @Nullable Domain domain, String name) + { + if (StringUtils.isEmpty(name)) + throw new ApiUsageException("List name is required."); + if (name.length() > MAX_NAME_LENGTH) + throw new ApiUsageException("List name cannot be longer than " + MAX_NAME_LENGTH + " characters."); + if (ListService.get().getList(container, name, domain == null) != null) + throw new ApiUsageException("The name '" + name + "' is already in use."); + } + + @Override + public Domain createDomain(GWTDomain domain, ListDomainKindProperties listProperties, Container container, User user, @Nullable TemplateInfo templateInfo, boolean forUpdate) + { + String name = StringUtils.trimToEmpty(domain.getName()); + validateDomainName(container, user, null, name); + + String keyName = listProperties.getKeyName(); + if (StringUtils.isEmpty(keyName)) + throw new ApiUsageException("List keyName is required"); + + KeyType keyType = getDefaultKeyType(); + + if (null != listProperties.getKeyType()) + { + String rawKeyType = listProperties.getKeyType(); + if (EnumUtils.isValidEnum(KeyType.class, rawKeyType)) + keyType = KeyType.valueOf(rawKeyType); + else + throw new ApiUsageException("List keyType provided does not exist."); + } + + ListDefinition.Category category; + try + { + category = listProperties.getCategory() != null ? ListDefinition.Category.valueOf(listProperties.getCategory()) : null; + } + catch (IllegalArgumentException e) + { + throw new ApiUsageException(String.format("List category type provided (%s) does not exist.", listProperties.getCategory())); + } + + if (!getSupportedKeyTypes().contains(keyType)) + throw new ApiUsageException("List keyType provided is not supported for list domain kind (" + getKindName() + ")."); + + ListDefinition list = ListService.get().createList(container, name, keyType, templateInfo, category); + list.setKeyName(keyName); + + // Issue 45042: Allow for the list description to be set via the create domain API calls + if (listProperties.getDescription() == null && domain.getDescription() != null) + listProperties.setDescription(domain.getDescription()); + list.setDescription(listProperties.getDescription()); + + List properties = (List)domain.getFields(); + List indices = (List)domain.getIndices(); + + try (DbScope.Transaction tx = ExperimentService.get().ensureTransaction()) + { + Domain d = list.getDomain(forUpdate); + + Set reservedNames = getReservedPropertyNames(d, user); + Set lowerReservedNames = reservedNames.stream().map(String::toLowerCase).collect(Collectors.toSet()); + + Map defaultValues = new HashMap<>(); + Set propertyUris = new CaseInsensitiveHashSet(); + for (GWTPropertyDescriptor pd : properties) + { + String propertyName = pd.getName().toLowerCase(); + if (lowerReservedNames.contains(propertyName)) + { + if (pd.getLabel() == null) + pd.setLabel(pd.getName()); + pd.setName("_" + pd.getName()); + } + + DomainUtil.addProperty(d, pd, defaultValues, propertyUris, null); + } + + d.setPropertyIndices(indices, lowerReservedNames); + + // TODO: This looks like the wrong order to me -- we should updateListProperties() (persist the indexing + // settings and handle potential transitions) before calling save() (which indexes the list). Since this is + // the create case there's no data to index, but there is meta data... + list.save(user, true, listProperties.getAuditRecordMap(), domain.getCalculatedFields()); + updateListProperties(container, user, list.getListId(), listProperties); + + QueryService.get().saveCalculatedFieldsMetadata(ListQuerySchema.NAME, name, null, domain.getCalculatedFields(), false, user, container); + + DefaultValueService.get().setDefaultValues(container, defaultValues); + + tx.commit(); + } + catch (Exception e) + { + throw new RuntimeException(e); + } + + return list.getDomain(forUpdate); + } + + @Override + public @NotNull ValidationException updateDomain(GWTDomain original, GWTDomain update, + ListDomainKindProperties listProperties, Container container, User user, boolean includeWarnings, @Nullable String auditUserComment) + { + ValidationException exception; + + try (DbScope.Transaction transaction = ListManager.get().getListMetadataSchema().getScope().ensureTransaction()) + { + exception = new ValidationException(); + StringBuilder changeDetails = new StringBuilder(); + + Domain domain = PropertyService.get().getDomain(container, original.getDomainURI()); + if (null == domain) + return exception.addGlobalError("Expected domain for list: " + original.getName()); + + ListDefinition listDefinition = ListService.get().getList(domain); + TableInfo table = listDefinition.getTable(user); + + if (null == table) + return exception.addGlobalError("Expected table for list: " + listDefinition.getName()); + + // Handle cases when existing key field is null or is not provided in the updated domainDesign + GWTPropertyDescriptor key = findField(listDefinition.getKeyName(), original.getFields()); + if (null != key) + { + int id = key.getPropertyId(); + GWTPropertyDescriptor newKey = findField(id, update.getFields()); + if (null == newKey) + { + return exception.addGlobalError("Key field not provided, expecting key field '" + key.getName() + "'"); + } + else if (!key.getName().equalsIgnoreCase(newKey.getName())) + { + return exception.addGlobalError("Cannot change key field name"); + } + } + else + { + return exception.addGlobalError("Key field not found for list '" + listDefinition.getName() + "'"); + } + + //handle name change + String updatedName = StringUtils.trim(update.getName()); + boolean hasNameChange = !original.getName().equals(updatedName); + if (hasNameChange) + { + validateDomainName(container, user, domain, updatedName); + changeDetails.append("The name of the list domain '" + original.getName() + "' was changed to '" + updatedName + "'."); + } + + //return if there are errors before moving forward with the save + if (exception.hasErrors()) + { + return exception; + } + + //update list properties + Map oldProps = null; + Map newProps = null; + if (null != listProperties) + { + if (listProperties.getDomainId() != original.getDomainId() || listProperties.getDomainId() != update.getDomainId()) + return exception.addGlobalError("domainId for the list does not match old or the new domain"); + if (!original.getDomainURI().equals(update.getDomainURI())) + return exception.addGlobalError("domainURI mismatch between old and new domain"); + + Pair, Map> updatedProps = updateListProperties(container, user, listDefinition.getListId(), listProperties); + oldProps = updatedProps.first; + newProps = updatedProps.second; + } + // Issue 45042: Allow for the list description to be set via the save domain API calls + else if (update.getDescription() != null) + { + listProperties = getListProperties(container, user, listDefinition.getListId()); + listProperties.setDescription(update.getDescription()); + Pair, Map> updatedProps = updateListProperties(container, user, listDefinition.getListId(), listProperties); + oldProps = updatedProps.first; + newProps = updatedProps.second; + } + + //update domain design properties + try + { + //handle attachment cols + Map modifiedAttachmentColumns = new CaseInsensitiveHashMap<>(); + + for (DomainProperty oldProp : domain.getProperties()) + { + if (PropertyType.ATTACHMENT.equals(oldProp.getPropertyDescriptor().getPropertyType())) + { + GWTPropertyDescriptor newGWTProp = findField(oldProp.getPropertyId(), update.getFields()); + if (null == newGWTProp || !PropertyType.ATTACHMENT.equals(PropertyType.getFromURI(newGWTProp.getConceptURI(), newGWTProp.getRangeURI(), null))) + { + ColumnInfo column = table.getColumn(oldProp.getPropertyDescriptor().getName()); + if (null != column) + modifiedAttachmentColumns.put(oldProp.getPropertyDescriptor().getName(), column); + } + } + } + + Collection> attachmentMapCollection = null; + if (!modifiedAttachmentColumns.isEmpty()) + { + List columns = new ArrayList<>(modifiedAttachmentColumns.values()); + columns.add(table.getColumn("entityId")); + attachmentMapCollection = new TableSelector(table, columns, null, null).getMapCollection(); + } + + // Remove attachments from any attachment columns that are removed or no longer attachment columns + if (null != attachmentMapCollection) + { + for (Map map : attachmentMapCollection) + { + String entityId = (String) map.get("entityId"); + ListItemAttachmentParent parent = new ListItemAttachmentParent(entityId, container); + for (Map.Entry entry : map.entrySet()) + if (null != entry.getValue() && modifiedAttachmentColumns.containsKey(entry.getKey())) + AttachmentService.get().deleteAttachment(parent, entry.getValue().toString(), user); + } + } + + //update domain properties + exception.addErrors(DomainUtil.updateDomainDescriptor(original, update, container, user, hasNameChange, changeDetails.toString(), auditUserComment, oldProps, newProps)); + + QueryService.get().saveCalculatedFieldsMetadata(ListQuerySchema.NAME, update.getQueryName(), hasNameChange ? update.getName() : null, update.getCalculatedFields(), !original.getCalculatedFields().isEmpty(), user, container); + } + catch (RuntimeSQLException x) + { + // issue 19202 - check for null value exceptions in case provided file data not contain the column + // and return a better error message + String message = x.getMessage(); + if (x.isNullValueException()) + { + message = "The provided data does not contain the specified '" + listDefinition.getKeyName() + "' field or contains null key values."; + } + return exception.addGlobalError(message); + } + catch (DataIntegrityViolationException | MetadataUnavailableException x) + { + return exception.addGlobalError("A data error occurred: " + x.getMessage()); + } + + if (!exception.hasErrors()) + { + transaction.commit(); + } + return exception; + } + } + + private ListDomainKindProperties getListProperties(Container container, User user, int listId) + { + SimpleFilter filter = SimpleFilter.createContainerFilter(container); + filter.addCondition(FieldKey.fromParts("ListId"), listId); + return new TableSelector(ListManager.get().getListMetadataTable(), filter, null).getObject(ListDomainKindProperties.class); + } + + private Pair, Map> updateListProperties(Container container, User user, int listId, ListDomainKindProperties listProperties) + { + ListDomainKindProperties existingListProps = getListProperties(container, user, listId); + + Map oldProps = existingListProps == null ? null : existingListProps.getAuditRecordMap(); + Map newProps = listProperties == null ? oldProps : listProperties.getAuditRecordMap(); + //merge existing and new properties + ListDomainKindProperties updatedListProps = updateListProperties(existingListProps, listProperties); + + ListManager.get().update(user, container, updatedListProps); + + return new Pair<>(oldProps, newProps); + } + + //updates list properties except listId, domainId, keyName, keyType, and lastIndexed + private ListDomainKindProperties updateListProperties(ListDomainKindProperties existingListProps, ListDomainKindProperties newListProps) + { + + ListDomainKindProperties updatedListProps = new ListDomainKindProperties(existingListProps); + if (null != newListProps.getName()) + updatedListProps.setName(newListProps.getName().trim()); + + updatedListProps.setTitleColumn(newListProps.getTitleColumn()); + updatedListProps.setDescription(newListProps.getDescription()); + updatedListProps.setAllowDelete(newListProps.isAllowDelete()); + updatedListProps.setAllowUpload(newListProps.isAllowUpload()); + updatedListProps.setAllowExport(newListProps.isAllowExport()); + updatedListProps.setCategory(newListProps.getCategory()); + updatedListProps.setEntireListTitleTemplate(newListProps.getEntireListTitleTemplate()); + updatedListProps.setEntireListIndexSetting(newListProps.getEntireListIndexSetting()); + updatedListProps.setEntireListBodySetting(newListProps.getEntireListBodySetting()); + updatedListProps.setEachItemTitleTemplate(newListProps.getEachItemTitleTemplate()); + updatedListProps.setEachItemBodySetting(newListProps.getEachItemBodySetting()); + updatedListProps.setEntireListIndex(newListProps.isEntireListIndex()); + updatedListProps.setEntireListBodyTemplate(newListProps.getEntireListBodyTemplate()); + updatedListProps.setEachItemIndex(newListProps.isEachItemIndex()); + updatedListProps.setEachItemBodyTemplate(newListProps.getEachItemBodyTemplate()); + updatedListProps.setFileAttachmentIndex(newListProps.isFileAttachmentIndex()); + + return updatedListProps; + } + + private GWTPropertyDescriptor findField(String name, List fields) + { + for (GWTPropertyDescriptor f : fields) + { + if (name.equalsIgnoreCase(f.getName())) + return f; + } + return null; + } + + private GWTPropertyDescriptor findField(int id, List fields) + { + if (id > 0) + { + for (GWTPropertyDescriptor f : fields) + { + if (id == f.getPropertyId()) + return f; + } + } + return null; + } + + @Override + public @Nullable ListDomainKindProperties getDomainKindProperties(GWTDomain domain, Container container, User user) + { + ListDefinition list = domain != null ? ListService.get().getList(PropertyService.get().getDomain(domain.getDomainId())) : null; + return ListManager.get().getListDomainKindProperties(container, list != null ? list.getListId() : null); + } + + @Override + public void deleteDomain(User user, Domain domain, String userComment) + { + ListDefinition list = ListService.get().getList(domain); + if (list == null) + throw new NotFoundException("List not found: " + domain.getTypeURI()); + + try + { + list.delete(user, userComment); + } + catch (DomainNotFoundException e) + { + throw new NotFoundException(e.getMessage()); + } + } + + @Override + public void invalidate(Domain domain) + { + super.invalidate(domain); + + ListDefinition list = ListService.get().getList(domain); + if (list != null) + ListManager.get().indexList(list); + } + + @Override + public boolean matchesTemplateXML(String templateName, DomainTemplateType template, List properties) + { + if (!(template instanceof ListTemplateType)) + return false; + + ListOptionsType options = ((ListTemplateType) template).getOptions(); + if (options == null) + throw new IllegalArgumentException("List template requires specifying a keyCol"); + + String keyName = options.getKeyCol(); + if (keyName == null) + throw new IllegalArgumentException("List template requires specifying a keyCol"); + + Pair pair = findProperty(templateName, properties, keyName); + GWTPropertyDescriptor prop = pair.first; + + PropertyType type = PropertyType.getFromURI(prop.getConceptURI(), prop.getRangeURI()); + + return type.equals(getDefaultKeyType().getPropertyType()); + } + + public void ensureBaseProperties(Domain d) + { + var props = getBaseProperties(d); + } + + @Override + public boolean supportsPhiLevel() + { + return ComplianceService.get().isComplianceSupported(); + } +} diff --git a/study/src/org/labkey/study/model/DatasetDomainKind.java b/study/src/org/labkey/study/model/DatasetDomainKind.java index c6b59b82ccb..c55fb877f22 100644 --- a/study/src/org/labkey/study/model/DatasetDomainKind.java +++ b/study/src/org/labkey/study/model/DatasetDomainKind.java @@ -1,872 +1,872 @@ -/* - * Copyright (c) 2008-2019 LabKey Corporation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.labkey.study.model; - -import org.apache.commons.beanutils.BeanUtils; -import org.apache.commons.lang3.StringUtils; -import org.jetbrains.annotations.NotNull; -import org.jetbrains.annotations.Nullable; -import org.labkey.api.action.ApiUsageException; -import org.labkey.api.collections.CaseInsensitiveHashSet; -import org.labkey.api.compliance.ComplianceService; -import org.labkey.api.data.BaseColumnInfo; -import org.labkey.api.data.ColumnInfo; -import org.labkey.api.data.Container; -import org.labkey.api.data.ContainerFilter; -import org.labkey.api.data.DbScope; -import org.labkey.api.data.JdbcType; -import org.labkey.api.data.PropertyStorageSpec; -import org.labkey.api.data.RuntimeSQLException; -import org.labkey.api.data.SQLFragment; -import org.labkey.api.data.SchemaTableInfo; -import org.labkey.api.data.TableInfo; -import org.labkey.api.di.DataIntegrationService; -import org.labkey.api.exp.Lsid; -import org.labkey.api.exp.PropertyDescriptor; -import org.labkey.api.exp.TemplateInfo; -import org.labkey.api.exp.api.ExperimentService; -import org.labkey.api.exp.api.StorageProvisioner; -import org.labkey.api.exp.property.AbstractDomainKind; -import org.labkey.api.exp.property.Domain; -import org.labkey.api.exp.property.DomainProperty; -import org.labkey.api.exp.property.DomainUtil; -import org.labkey.api.exp.property.PropertyService; -import org.labkey.api.gwt.client.model.GWTDomain; -import org.labkey.api.gwt.client.model.GWTIndex; -import org.labkey.api.gwt.client.model.GWTPropertyDescriptor; -import org.labkey.api.query.MetadataUnavailableException; -import org.labkey.api.query.QueryService; -import org.labkey.api.query.ValidationException; -import org.labkey.api.reports.model.ViewCategory; -import org.labkey.api.reports.model.ViewCategoryManager; -import org.labkey.api.security.User; -import org.labkey.api.security.permissions.AdminPermission; -import org.labkey.api.study.Dataset; -import org.labkey.api.study.Dataset.KeyManagementType; -import org.labkey.api.study.Study; -import org.labkey.api.study.StudyService; -import org.labkey.api.study.TimepointType; -import org.labkey.api.util.UnexpectedException; -import org.labkey.api.view.ActionURL; -import org.labkey.api.view.NotFoundException; -import org.labkey.api.writer.ContainerUser; -import org.labkey.study.StudySchema; -import org.labkey.study.assay.StudyPublishManager; -import org.labkey.study.controllers.StudyController; -import org.labkey.study.query.StudyQuerySchema; - -import java.util.ArrayList; -import java.util.Arrays; -import java.util.Collections; -import java.util.HashMap; -import java.util.HashSet; -import java.util.List; -import java.util.Map; -import java.util.Objects; -import java.util.Set; -import java.util.concurrent.locks.Lock; -import java.util.stream.Collectors; - -import static org.labkey.api.util.IntegerUtils.asInteger; -import static org.labkey.study.model.DatasetDomainKindProperties.TIME_KEY_FIELD_KEY; - -public abstract class DatasetDomainKind extends AbstractDomainKind -{ - public final static String LSID_PREFIX = "StudyDataset"; - - public final static String CONTAINER = "container"; - public final static String DATE = "date"; - public final static String PARTICIPANTID = "participantid"; - public final static String LSID = "lsid"; - public final static String DSROWID = "dsrowid"; - public final static String SEQUENCENUM = "sequencenum"; - public final static String SOURCELSID = "sourcelsid"; - public final static String _KEY = "_key"; - public final static String QCSTATE = "qcstate"; - public final static String PARTICIPANTSEQUENCENUM = "participantsequencenum"; - - public static final String CREATED = "created"; - public static final String MODIFIED = "modified"; - public static final String CREATED_BY = "createdBy"; - public static final String MODIFIED_BY = "modifiedBy"; - - - /* - * the columns common to all datasets - */ - private final static Set BASE_PROPERTIES; - private final static Set DATASPACE_BASE_PROPERTIES; - protected final static Set PROPERTY_INDICES; - private final static Set DATASPACE_PROPERTY_INDICES; - - static - { - DATASPACE_BASE_PROPERTIES = new HashSet<>(Arrays.asList( - new PropertyStorageSpec(DSROWID, JdbcType.BIGINT, 0, PropertyStorageSpec.Special.PrimaryKeyNonClustered, false, true, null), - new PropertyStorageSpec(CONTAINER, JdbcType.GUID).setNullable(false), - new PropertyStorageSpec(PARTICIPANTID, JdbcType.VARCHAR, 32).setNullable(false), - new PropertyStorageSpec(LSID, JdbcType.VARCHAR, 200), - new PropertyStorageSpec(SEQUENCENUM, JdbcType.DECIMAL), - new PropertyStorageSpec(SOURCELSID, JdbcType.VARCHAR, 200), - new PropertyStorageSpec(_KEY, JdbcType.VARCHAR, 200), - new PropertyStorageSpec(QCSTATE, JdbcType.INTEGER), - new PropertyStorageSpec(PARTICIPANTSEQUENCENUM, JdbcType.VARCHAR, 200), - new PropertyStorageSpec(CREATED, JdbcType.TIMESTAMP), - new PropertyStorageSpec(MODIFIED, JdbcType.TIMESTAMP), - new PropertyStorageSpec(CREATED_BY, JdbcType.INTEGER), - new PropertyStorageSpec(MODIFIED_BY, JdbcType.INTEGER), - new PropertyStorageSpec(DATE, JdbcType.TIMESTAMP), - new PropertyStorageSpec(DataIntegrationService.Columns.TransformImportHash.getColumnName(), JdbcType.VARCHAR, 256) - )); - - - BASE_PROPERTIES = new HashSet<>(Arrays.asList( - new PropertyStorageSpec(DSROWID, JdbcType.BIGINT, 0, PropertyStorageSpec.Special.PrimaryKeyNonClustered, false, true, null), - new PropertyStorageSpec(PARTICIPANTID, JdbcType.VARCHAR, 32), - new PropertyStorageSpec(LSID, JdbcType.VARCHAR, 200), - new PropertyStorageSpec(SEQUENCENUM, JdbcType.DECIMAL), - new PropertyStorageSpec(SOURCELSID, JdbcType.VARCHAR, 200), - new PropertyStorageSpec(_KEY, JdbcType.VARCHAR, 200), - new PropertyStorageSpec(QCSTATE, JdbcType.INTEGER), - new PropertyStorageSpec(PARTICIPANTSEQUENCENUM, JdbcType.VARCHAR, 200), - new PropertyStorageSpec(CREATED, JdbcType.TIMESTAMP), - new PropertyStorageSpec(MODIFIED, JdbcType.TIMESTAMP), - new PropertyStorageSpec(CREATED_BY, JdbcType.INTEGER), - new PropertyStorageSpec(MODIFIED_BY, JdbcType.INTEGER), - new PropertyStorageSpec(DATE, JdbcType.TIMESTAMP), - new PropertyStorageSpec(DataIntegrationService.Columns.TransformImportHash.getColumnName(), JdbcType.VARCHAR, 256) - )); - - DATASPACE_PROPERTY_INDICES = new HashSet<>(Arrays.asList( - new PropertyStorageSpec.Index(false, true, CONTAINER, PARTICIPANTID, DATE), - new PropertyStorageSpec.Index(false, CONTAINER, QCSTATE), - new PropertyStorageSpec.Index(true, CONTAINER, PARTICIPANTID, SEQUENCENUM, _KEY), - new PropertyStorageSpec.Index(true, LSID), - new PropertyStorageSpec.Index(false, DATE) - )); - - PROPERTY_INDICES = new HashSet<>(Arrays.asList( - new PropertyStorageSpec.Index(false, true, PARTICIPANTID, DATE), - new PropertyStorageSpec.Index(false, QCSTATE), - new PropertyStorageSpec.Index(true, PARTICIPANTID, SEQUENCENUM, _KEY), - new PropertyStorageSpec.Index(true, LSID), - new PropertyStorageSpec.Index(false, DATE) - )); - } - - - protected DatasetDomainKind() - { - } - - @Override - abstract public String getKindName(); - - @Override - public Class getTypeClass() - { - return DatasetDomainKindProperties.class; - } - - @Override - public String getTypeLabel(Domain domain) - { - DatasetDefinition def = getDatasetDefinition(domain.getTypeURI()); - if (null == def) - return domain.getName(); - return def.getName(); - } - - @Override - public SQLFragment sqlObjectIdsInDomain(Domain domain) - { - DatasetDefinition def = getDatasetDefinition(domain.getTypeURI()); - if (null == def) - return new SQLFragment("NULL"); - TableInfo ti = def.getStorageTableInfo(false); - SQLFragment sql = new SQLFragment(); - sql.append("SELECT O.ObjectId FROM ").append(ti).append(" SD JOIN exp.Object O ON SD.Lsid=O.ObjectURI WHERE O.container=?"); - sql.add(def.getContainer()); - return sql; - } - - // Issue 16526: nobody should call this overload of generateDomainURI for DatasetDomainKind. Instead - // use the overload below with a unique id (the dataset's entityId). Assert is here to track down - // any callers. - // Lsid.toString() encodes incorrectly TODO: fix - @Override - public String generateDomainURI(String schemaName, String name, Container container, User user) - { - assert false; - return null; - } - - // Issue 16526: This specific generateDomainURI takes an id to uniquify the dataset. - public static String generateDomainURI(String name, String id, Container container) - { - String objectid = name == null ? "" : name; - if (null != id) - { - // normalize the object id - objectid += "-" + id.toLowerCase(); - } - return (new Lsid(LSID_PREFIX, "Folder-" + container.getRowId(), objectid)).toString(); - } - - @Override - public ActionURL urlShowData(Domain domain, ContainerUser containerUser) - { - Dataset def = getDatasetDefinition(domain.getTypeURI()); - ActionURL url = new ActionURL(StudyController.DatasetReportAction.class, containerUser.getContainer()); - url.addParameter("datasetId", "" + def.getDatasetId()); - return url; - } - - @Override - public ActionURL urlEditDefinition(Domain domain, ContainerUser containerUser) - { - Dataset def = getDatasetDefinition(domain.getTypeURI()); - ActionURL url = new ActionURL(StudyController.EditTypeAction.class, containerUser.getContainer()); - url.addParameter("datasetId", "" + def.getDatasetId()); - return url; - } - - @Override - public ActionURL urlCreateDefinition(String schemaName, String queryName, Container container, User user) - { - return new ActionURL(StudyController.DefineDatasetTypeAction.class, container); - } - - @Override - public boolean allowFileLinkProperties() - { - return true; - } - - @Override - public boolean allowMultiChoiceProperties() - { - return true; - } - - @Override - public boolean showDefaultValueSettings() - { - return true; - } - - @Override - public boolean allowUniqueConstraintProperties() - { - return true; - } - - @Override - public boolean allowCalculatedFields() - { - return true; - } - - DatasetDefinition getDatasetDefinition(String domainURI) - { - return StudyManager.getInstance().getDatasetDefinition(domainURI); - } - - // Issue 43898: Add the study subject name column to reserved fields - protected Set getStudySubjectReservedName(Domain domain) - { - HashSet fields = new CaseInsensitiveHashSet(); - if (null != domain) - { - Study study = StudyManager.getInstance().getStudy(domain.getContainer()); - if (null != study) - { - String participantIdField = study.getSubjectColumnName(); - fields.addAll(DomainUtil.getNameAndLabels(participantIdField)); - } - } - return Collections.unmodifiableSet(fields); - } - - @Override - public abstract @NotNull Set getReservedPropertyNames(Domain domain, User user); - - @Override - public Set getBaseProperties(Domain domain) - { - Set specs; - Study study = StudyManager.getInstance().getStudy(domain.getContainer()); - - if(study == null || study.isDataspaceStudy()) - { - specs = new HashSet<>(DATASPACE_BASE_PROPERTIES); - } - else - { - specs = new HashSet<>(BASE_PROPERTIES); - } - specs.addAll(super.getBaseProperties(domain)); - return specs; - } - - @Override - public Set getPropertyIndices(Domain domain) - { - Study study = StudyManager.getInstance().getStudy(domain.getContainer()); - - if(study == null || study.isDataspaceStudy()) - return DATASPACE_PROPERTY_INDICES; - - return PROPERTY_INDICES; - } - - @Override - public DbScope getScope() - { - return StudySchema.getInstance().getSchema().getScope(); - } - - @Override - public String getStorageSchemaName() - { - return StudySchema.getInstance().getDatasetSchemaName(); - } - - @Override - public void invalidate(Domain domain) - { - super.invalidate(domain); - DatasetDefinition def = getDatasetDefinition(domain.getTypeURI()); - if (null != def) - { - StudyManager.getInstance().uncache(def); - } - } - - @Override - public boolean canCreateDefinition(User user, Container container) - { - return container.hasPermission(user, AdminPermission.class); - } - - @Override - public Map processArguments(Container container, User user, Map arguments) - { - Map updatedArguments = new HashMap<>(arguments); - - // For backwards compatibility, map "demographics" => "demographicData" - if (arguments.containsKey("demographics")) - { - updatedArguments.put("demographicData", arguments.get("demographics")); - updatedArguments.remove("demographics"); - } - - // For backwards compatibility, map "categoryId" and "categoryName" => "category" - if (arguments.containsKey("categoryId")) - { - if (arguments.containsKey("categoryName")) - throw new IllegalArgumentException("Category ID and category name cannot both be specified."); - - ViewCategory category = ViewCategoryManager.getInstance().getCategory(container, asInteger(arguments.get("categoryId"))); - if (category == null) - throw new IllegalArgumentException("Unable to find a category with the ID : " + arguments.get("categoryId") + " in this folder."); - - updatedArguments.put("category", category.getLabel()); - updatedArguments.remove("categoryId"); - } - else if (arguments.containsKey("categoryName")) - { - updatedArguments.put("category", arguments.get("categoryName")); - updatedArguments.remove("categoryName"); - } - - return updatedArguments; - } - - @Nullable - @Override - public DatasetDomainKindProperties getDomainKindProperties(GWTDomain domain, Container container, User user) - { - Dataset ds = domain != null ? getDatasetDefinition(domain.getDomainURI()) : null; - return DatasetManager.get().getDatasetDomainKindProperties(container, ds != null ? ds.getDatasetId() : null); - } - - @Override - public Domain createDomain(GWTDomain domain, DatasetDomainKindProperties arguments, Container container, User user, - @Nullable TemplateInfo templateInfo, boolean forUpdate) - { - arguments.setName(StringUtils.trimToNull(domain.getName())); - String name = arguments.getName(); - if (name == null) - throw new IllegalArgumentException("Dataset name cannot be empty."); - String description = arguments.getDescription() != null ? arguments.getDescription() : domain.getDescription(); - String label = (arguments.getLabel() == null || arguments.getLabel().isEmpty()) ? arguments.getName() : arguments.getLabel(); - Integer cohortId = arguments.getCohortId(); - String tag = arguments.getTag(); - Integer datasetId = arguments.getDatasetId(); - String categoryName = arguments.getCategory(); - boolean demographics = arguments.isDemographicData(); - boolean isManagedField = arguments.isKeyPropertyManaged(); - String visitDatePropertyName = arguments.getVisitDatePropertyName(); - boolean useTimeKeyField = arguments.isUseTimeKeyField(); - boolean showByDefault = arguments.isShowByDefault(); - String dataSharing = arguments.getDataSharing(); - - // general dataset validation - validateDatasetProperties(arguments, container, user, domain, null); - - // create-case specific validation - StudyImpl study = StudyManager.getInstance().getStudy(container); - TimepointType timepointType = study.getTimepointType(); - if (timepointType.isVisitBased() && getKindName().equals(DateDatasetDomainKind.KIND_NAME)) - throw new IllegalArgumentException("Visit based studies require a visit based dataset domain. Please specify a kind name of : " + VisitDatasetDomainKind.KIND_NAME + "."); - else if (!timepointType.isVisitBased() && getKindName().equals(VisitDatasetDomainKind.KIND_NAME)) - throw new IllegalArgumentException("Date based studies require a date based dataset domain. Please specify a kind name of : " + DateDatasetDomainKind.KIND_NAME + "."); - if (timepointType.isVisitBased() && useTimeKeyField) - throw new IllegalArgumentException("Additional key property cannot be Time (from Date/Time) for visit based studies."); - - // Check for usage of Time as Key Field - String keyPropertyName = arguments.getKeyPropertyName(); - if (useTimeKeyField) - keyPropertyName = null; - - try (DbScope.Transaction transaction = ExperimentService.get().ensureTransaction()) - { - KeyManagementType managementType = KeyManagementType.None; - if (isManagedField) - { - String rangeUri = ""; - for (GWTPropertyDescriptor a : domain.getFields()) - { - if (keyPropertyName.equalsIgnoreCase(a.getName())) - { - rangeUri = a.getRangeURI(); - break; - } - - } - - PropertyDescriptor pd = new PropertyDescriptor(); - pd.setRangeURI(rangeUri); - managementType = KeyManagementType.getManagementTypeFromProp(pd.getPropertyType()); - } - - Integer categoryId = null; - ViewCategory category; - if (categoryName != null) - { - category = ViewCategoryManager.getInstance().getCategory(container, categoryName); - if (category != null) - { - categoryId = category.getRowId(); - } - else - { - String[] parts = ViewCategoryManager.getInstance().decode(categoryName); - category = ViewCategoryManager.getInstance().ensureViewCategory(container, user, parts); - categoryId = category.getRowId(); - } - } - - DatasetDefinition def = StudyPublishManager.getInstance().createDataset(user, new DatasetDefinition.Builder(name) - .setStudy(study) - .setKeyPropertyName(keyPropertyName) - .setDatasetId(datasetId) - .setDemographicData(demographics) - .setCategoryId(categoryId) - .setUseTimeKeyField(useTimeKeyField) - .setKeyManagementType(managementType) - .setShowByDefault(showByDefault) - .setLabel(label) - .setDescription(description) - .setCohortId(cohortId) - .setTag(tag) - .setVisitDatePropertyName(visitDatePropertyName) - .setDataSharing(dataSharing)); - - if (def.getDomain() != null) - { - List properties = domain.getFields(); - - Domain newDomain = def.getDomain(true); - if (newDomain != null) - { - Set reservedNames = getReservedPropertyNames(newDomain, user); - Set lowerReservedNames = reservedNames.stream().map(String::toLowerCase).collect(Collectors.toSet()); - Set existingProperties = newDomain.getProperties().stream().map(o -> o.getName().toLowerCase()).collect(Collectors.toSet()); - Map defaultValues = new HashMap<>(); - Set propertyUris = new CaseInsensitiveHashSet(); - - for (GWTPropertyDescriptor pd : properties) - { - if (lowerReservedNames.contains(pd.getName().toLowerCase()) || existingProperties.contains(pd.getName().toLowerCase())) - { - if (arguments.isStrictFieldValidation()) - throw new ApiUsageException("Property: " + pd.getName() + " is reserved or exists in the current domain."); - } - else - DomainUtil.addProperty(newDomain, pd, defaultValues, propertyUris, null); - } - - newDomain.save(user, arguments.getAuditRecordMap(), domain.getCalculatedFields()); - - List indices = domain.getIndices(); - newDomain.setPropertyIndices(indices, lowerReservedNames); - StorageProvisioner.get().ensureTableIndices(newDomain); - - QueryService.get().saveCalculatedFieldsMetadata("study", name, null, domain.getCalculatedFields(), false, user, container); - } - else - throw new IllegalArgumentException("Failed to create domain for dataset : " + name + "."); - } - - transaction.commit(); - return study.getDataset(def.getDatasetId()).getDomain(forUpdate); - } - catch (Exception e) - { - UnexpectedException.rethrow(e); // don't re-wrap runtime exceptions - return null; // can't get here - } - } - - private void validateDatasetProperties(DatasetDomainKindProperties datasetProperties, Container container, User user, GWTDomain domain, DatasetDefinition def) - { - String name = StringUtils.trimToEmpty(datasetProperties.getName()); - String keyPropertyName = datasetProperties.getKeyPropertyName(); - Integer datasetId = datasetProperties.getDatasetId(); - boolean isManagedField = datasetProperties.isKeyPropertyManaged(); - boolean useTimeKeyField = datasetProperties.isUseTimeKeyField(); - boolean isDemographicData = datasetProperties.isDemographicData(); - - if (!container.hasPermission(user, AdminPermission.class)) - throw new IllegalArgumentException("You do not have permissions to edit dataset definitions in this container."); - - Study study = StudyService.get().getStudy(container); - if (study == null) - throw new IllegalArgumentException("A study does not exist for this container."); - - // Name related exceptions - - if (name == null || name.isEmpty()) - throw new IllegalArgumentException("Dataset name cannot be empty."); - - if (name.length() > 200) - throw new IllegalArgumentException("Dataset name must be under 200 characters."); - - if (!name.equals(domain.getName()) || def == null) - { - // issue 17766: check if dataset or query exist with this name - if (null != StudyManager.getInstance().getDatasetDefinitionByName(study, name) || null != QueryService.get().getQueryDef(user, container, "study", name)) - throw new IllegalArgumentException("A Dataset or Query already exists with the name \"" + name + "\"."); - - StudyQuerySchema schema = StudyQuerySchema.createSchema(StudyManager.getInstance().getStudy(container), User.getSearchUser()); - - // Now check standard study tables - if (schema.getTableNames().contains(name)) - throw new IllegalArgumentException("A study table exists with the name \"" + name + "\"."); - - String datasetNameError = DomainUtil.validateDomainName(name, getKindName(), false); - if (datasetNameError != null) - throw new IllegalArgumentException(datasetNameError); - } - - // Label related exceptions - - String label = Objects.requireNonNullElse(datasetProperties.getLabel(), name); // Need to verify label uniqueness even if label is unspecified - - if ((def == null || !def.getLabel().equals(label)) && null != StudyManager.getInstance().getDatasetDefinitionByLabel(study, label)) - throw new IllegalArgumentException("A Dataset already exists with the label \"" + label +"\"."); - - // Additional key related exceptions - - if ("".equals(keyPropertyName)) - throw new IllegalArgumentException("Please select a field name for the additional key."); - - if (isManagedField && (keyPropertyName == null || keyPropertyName.isEmpty())) - throw new IllegalArgumentException("Additional Key Column name must be specified if field is managed."); - - if (useTimeKeyField && isManagedField) - throw new IllegalArgumentException("Additional key cannot be a managed field if KeyPropertyName is Time (from Date/Time)."); - - if (useTimeKeyField && !(keyPropertyName == null || keyPropertyName.equals(TIME_KEY_FIELD_KEY))) - throw new IllegalArgumentException("KeyPropertyName should not be provided when using additional key of Time (from Date/Time)."); - - if (isDemographicData && (isManagedField || keyPropertyName != null)) - throw new IllegalArgumentException("There cannot be an Additional Key Column if the dataset is Demographic Data."); - - if (!useTimeKeyField && null != keyPropertyName && null == domain.getFieldByName(keyPropertyName)) - throw new IllegalArgumentException("Additional Key Column name \"" + keyPropertyName +"\" must be the name of a column."); - - if (null != keyPropertyName && isManagedField) - { - String rangeURI = domain.getFieldByName(keyPropertyName).getRangeURI(); - if (!(rangeURI.endsWith("int") || rangeURI.endsWith("double") || rangeURI.endsWith("string"))) - throw new IllegalArgumentException("If Additional Key Column is managed, the column type must be numeric or text-based."); - } - - // Other exception(s) - - if (null != datasetId && (null == def || def.getDatasetId() != datasetId) && null != study.getDataset(datasetId)) - throw new IllegalArgumentException("A Dataset already exists with the datasetId \"" + datasetId +"\"."); - - if (!study.getShareVisitDefinitions() && null != datasetProperties.getDataSharing() && !datasetProperties.getDataSharing().equals("NONE")) - throw new IllegalArgumentException("Illegal value set for data sharing option."); - } - - private void checkCanUpdate(DatasetDefinition def, Container container, User user, DatasetDomainKindProperties datasetProperties, - GWTDomain original, GWTDomain update) - { - if (null == def) - throw new IllegalArgumentException("Dataset not found."); - - if (!def.canUpdateDefinition(user)) - throw new IllegalArgumentException("Shared dataset can not be edited in this folder."); - - if (datasetProperties.getLabel() == null || datasetProperties.getLabel().isEmpty()) - throw new IllegalArgumentException("Dataset label cannot be empty."); - - if (null == PropertyService.get().getDomain(container, update.getDomainURI())) - throw new IllegalArgumentException("Domain not found: " + update.getDomainURI() + "."); - - if (!def.getTypeURI().equals(original.getDomainURI()) || !def.getTypeURI().equals(update.getDomainURI())) - throw new IllegalArgumentException("Illegal Argument"); - - if (datasetProperties.isDemographicData() && !def.isDemographicData() && !StudyManager.getInstance().isDataUniquePerParticipant(def)) - { - String noun = StudyService.get().getSubjectNounSingular(container); - throw new IllegalArgumentException("This dataset currently contains more than one row of data per " + noun + - ". Demographic data includes one row of data per " + noun + "."); - } - } - - private @NotNull ValidationException updateDomainDescriptor(GWTDomain original, GWTDomain update, - @Nullable DatasetDefinition oldDef, @Nullable DatasetDomainKindProperties datasetPropertiesUpdate, Container container, User user, String userComment) - { - StringBuilder changeDetails = new StringBuilder(); - boolean hasNameChange = false; - Map oldProps = null; - Map newProps = datasetPropertiesUpdate != null ? datasetPropertiesUpdate.getAuditRecordMap() : null; - - if (oldDef != null) - { - oldProps = oldDef.getAuditRecordMap(); - if (newProps == null) - newProps = oldProps; // no update - - hasNameChange = !datasetPropertiesUpdate.getName().equals(oldDef.getName()); - if (hasNameChange) - changeDetails.append("The name of the dataset '" + oldDef.getName() + "' was changed to '" + datasetPropertiesUpdate.getName() + "'."); - } - - ValidationException exception = new ValidationException(); - exception.addErrors(DomainUtil.updateDomainDescriptor(original, update, container, user, hasNameChange, changeDetails.toString(), userComment, oldProps, newProps)); - return exception; - } - - private ValidationException updateDataset(DatasetDomainKindProperties datasetProperties, String domainURI, ValidationException exception, - StudyImpl study, Container container, User user, DatasetDefinition def) - { - try - { - // Check for usage of Time as Key Field - boolean useTimeKeyField = datasetProperties.isUseTimeKeyField(); - if (useTimeKeyField) - datasetProperties.setKeyPropertyName(null); - - // Default is no key management - KeyManagementType keyType = KeyManagementType.None; - String keyPropertyName = datasetProperties.getKeyPropertyName(); - if (datasetProperties.getKeyPropertyName() != null) - { - Domain domain = PropertyService.get().getDomain(container, domainURI); - for (DomainProperty dp : domain.getProperties()) - { - if (dp.getName().equalsIgnoreCase(datasetProperties.getKeyPropertyName())) - { - keyPropertyName = dp.getName(); - - // Be sure that the user really wants a managed key, not just that disabled select box still had a value - if (datasetProperties.isKeyPropertyManaged()) - keyType = KeyManagementType.getManagementTypeFromProp(dp.getPropertyDescriptor().getPropertyType()); - - break; - } - } - } - - DatasetDefinition updated = def.createMutable(); - - // Clear the category ID so that it gets regenerated based on the new string - see issue 19649 - updated.setCategoryId(null); - - BeanUtils.copyProperties(updated, datasetProperties); - updated.setKeyPropertyName(keyPropertyName); - updated.setKeyManagementType(keyType); - updated.setUseTimeKeyField(useTimeKeyField); - - List errors = new ArrayList<>(); - StudyManager.getInstance().updateDatasetDefinition(user, updated, errors); - StudyManager.datasetModified(updated, true); - - for (String errorMsg: errors) - exception.addGlobalError(errorMsg); - return exception; - } - catch (RuntimeSQLException e) - { - return exception.addGlobalError("Additional key column must have unique values."); - } - catch (Exception e) - { - throw new RuntimeException(e); - } - } - - @Override - public @NotNull ValidationException updateDomain(GWTDomain original, GWTDomain update, - @Nullable DatasetDomainKindProperties datasetProperties, Container container, User user, boolean includeWarnings, String userComment) - { - assert original.getDomainURI().equals(update.getDomainURI()); - StudyImpl study = StudyManager.getInstance().getStudy(container); - DatasetDefinition def = null; - boolean hasNameChange = false; - - if (datasetProperties != null) - { - def = study.getDataset(datasetProperties.getDatasetId()); - validateDatasetProperties(datasetProperties, container, user, update, def); - checkCanUpdate(def, container, user, datasetProperties, original, update); - String updatedName = StringUtils.trimToEmpty(datasetProperties.getName()); - datasetProperties.setName(updatedName); - hasNameChange = !def.getName().equals(updatedName); - } - - // Acquire lock before we actually start the transaction to avoid deadlocks when it's refreshed during the process - Lock[] locks = def == null ? new Lock[0] : new Lock[] { def.getDomainLoadingLock() }; - try (DbScope.Transaction transaction = StudySchema.getInstance().getScope().ensureTransaction(locks)) - { - ValidationException exception = updateDomainDescriptor(original, update, def, datasetProperties, container, user, userComment); - - QueryService.get().saveCalculatedFieldsMetadata("study", update.getQueryName(), hasNameChange ? datasetProperties.getName() : null, update.getCalculatedFields(), !original.getCalculatedFields().isEmpty(), user, container); - - if (!exception.hasErrors() && def != null) - exception = updateDataset(datasetProperties, original.getDomainURI(), exception, study, container, user, def); - - if (!exception.hasErrors()) - transaction.commit(); - - return exception; - } - catch (MetadataUnavailableException e) - { - return new ValidationException(e.getMessage()); - } - finally - { - if (def != null) - StudyManager.getInstance().uncache(def); - } - } - - @Override - public void deleteDomain(User user, Domain domain, @Nullable String auditUserComment) - { - DatasetDefinition def = StudyManager.getInstance().getDatasetDefinition(domain.getTypeURI()); - if (def == null) - throw new NotFoundException("Dataset not found: " + domain.getTypeURI()); - - StudyImpl study = StudyManager.getInstance().getStudy(domain.getContainer()); - if (study == null) - throw new IllegalArgumentException("A study does not exist for this folder"); - - try (DbScope.Transaction transaction = StudySchema.getInstance().getSchema().getScope().ensureTransaction()) - { - StudyManager.getInstance().deleteDataset(study, user, def, false, auditUserComment); - transaction.commit(); - } - } - - @Override - public boolean isDeleteAllDataOnFieldImport() - { - return true; - } - - - @Override - public TableInfo getTableInfo(User user, Container container, String name, @Nullable ContainerFilter cf) - { - StudyImpl study = StudyManager.getInstance().getStudy(container); - if (null == study) - return null; - StudyQuerySchema schema = StudyQuerySchema.createSchema(study, user); - DatasetDefinition dsd = schema.getDatasetDefinitionByName(name); - if (null == dsd) - return null; - - return schema.getTable(name, cf, true, false); - } - - @Override - public void afterLoadTable(SchemaTableInfo ti, Domain domain) - { - // Grab the "standard" properties and apply them to this dataset table - TableInfo template = DatasetDefinition.getTemplateTableInfo(); - - for (PropertyStorageSpec pss : domain.getDomainKind().getBaseProperties(domain)) - { - ColumnInfo c = ti.getColumn(pss.getName()); - ColumnInfo tCol = template.getColumn(pss.getName()); - // The column may be null if the dataset is being deleted in the background - if (null != tCol && c != null) - { - ((BaseColumnInfo)c).setExtraAttributesFrom(tCol); - - // When copying a column, the hidden bit is not propagated, so we need to do it manually - if (tCol.isHidden()) - ((BaseColumnInfo)c).setHidden(true); - } - } - } - - @Override - public boolean supportsPhiLevel() - { - return ComplianceService.get().isComplianceSupported(); - } - - // Query datasets don't have a domain kind, so check here if the dataset is a query dataset - @Override - public boolean isProvisioned(Container container, String name) - { - return super.isProvisioned(container, name) && !isQueryDataset(container, name); - } - - public static boolean isQueryDataset(Container container, String queryName) - { - StudyService ss = StudyService.get(); - if (ss == null) - return false; - - Dataset dt = ss.getDataset(container, ss.getDatasetIdByName(container, queryName)); - if (dt == null) - return false; - - return dt.isQueryDataset(); - } -} +/* + * Copyright (c) 2008-2019 LabKey Corporation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.labkey.study.model; + +import org.apache.commons.beanutils.BeanUtils; +import org.apache.commons.lang3.StringUtils; +import org.jetbrains.annotations.NotNull; +import org.jetbrains.annotations.Nullable; +import org.labkey.api.action.ApiUsageException; +import org.labkey.api.collections.CaseInsensitiveHashSet; +import org.labkey.api.compliance.ComplianceService; +import org.labkey.api.data.BaseColumnInfo; +import org.labkey.api.data.ColumnInfo; +import org.labkey.api.data.Container; +import org.labkey.api.data.ContainerFilter; +import org.labkey.api.data.DbScope; +import org.labkey.api.data.JdbcType; +import org.labkey.api.data.PropertyStorageSpec; +import org.labkey.api.data.RuntimeSQLException; +import org.labkey.api.data.SQLFragment; +import org.labkey.api.data.SchemaTableInfo; +import org.labkey.api.data.TableInfo; +import org.labkey.api.di.DataIntegrationService; +import org.labkey.api.exp.Lsid; +import org.labkey.api.exp.PropertyDescriptor; +import org.labkey.api.exp.TemplateInfo; +import org.labkey.api.exp.api.ExperimentService; +import org.labkey.api.exp.api.StorageProvisioner; +import org.labkey.api.exp.property.AbstractDomainKind; +import org.labkey.api.exp.property.Domain; +import org.labkey.api.exp.property.DomainProperty; +import org.labkey.api.exp.property.DomainUtil; +import org.labkey.api.exp.property.PropertyService; +import org.labkey.api.gwt.client.model.GWTDomain; +import org.labkey.api.gwt.client.model.GWTIndex; +import org.labkey.api.gwt.client.model.GWTPropertyDescriptor; +import org.labkey.api.query.MetadataUnavailableException; +import org.labkey.api.query.QueryService; +import org.labkey.api.query.ValidationException; +import org.labkey.api.reports.model.ViewCategory; +import org.labkey.api.reports.model.ViewCategoryManager; +import org.labkey.api.security.User; +import org.labkey.api.security.permissions.AdminPermission; +import org.labkey.api.study.Dataset; +import org.labkey.api.study.Dataset.KeyManagementType; +import org.labkey.api.study.Study; +import org.labkey.api.study.StudyService; +import org.labkey.api.study.TimepointType; +import org.labkey.api.util.UnexpectedException; +import org.labkey.api.view.ActionURL; +import org.labkey.api.view.NotFoundException; +import org.labkey.api.writer.ContainerUser; +import org.labkey.study.StudySchema; +import org.labkey.study.assay.StudyPublishManager; +import org.labkey.study.controllers.StudyController; +import org.labkey.study.query.StudyQuerySchema; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.Set; +import java.util.concurrent.locks.Lock; +import java.util.stream.Collectors; + +import static org.labkey.api.util.IntegerUtils.asInteger; +import static org.labkey.study.model.DatasetDomainKindProperties.TIME_KEY_FIELD_KEY; + +public abstract class DatasetDomainKind extends AbstractDomainKind +{ + public final static String LSID_PREFIX = "StudyDataset"; + + public final static String CONTAINER = "container"; + public final static String DATE = "date"; + public final static String PARTICIPANTID = "participantid"; + public final static String LSID = "lsid"; + public final static String DSROWID = "dsrowid"; + public final static String SEQUENCENUM = "sequencenum"; + public final static String SOURCELSID = "sourcelsid"; + public final static String _KEY = "_key"; + public final static String QCSTATE = "qcstate"; + public final static String PARTICIPANTSEQUENCENUM = "participantsequencenum"; + + public static final String CREATED = "created"; + public static final String MODIFIED = "modified"; + public static final String CREATED_BY = "createdBy"; + public static final String MODIFIED_BY = "modifiedBy"; + + + /* + * the columns common to all datasets + */ + private final static Set BASE_PROPERTIES; + private final static Set DATASPACE_BASE_PROPERTIES; + protected final static Set PROPERTY_INDICES; + private final static Set DATASPACE_PROPERTY_INDICES; + + static + { + DATASPACE_BASE_PROPERTIES = new HashSet<>(Arrays.asList( + new PropertyStorageSpec(DSROWID, JdbcType.BIGINT, 0, PropertyStorageSpec.Special.PrimaryKeyNonClustered, false, true, null), + new PropertyStorageSpec(CONTAINER, JdbcType.GUID).setNullable(false), + new PropertyStorageSpec(PARTICIPANTID, JdbcType.VARCHAR, 32).setNullable(false), + new PropertyStorageSpec(LSID, JdbcType.VARCHAR, 200), + new PropertyStorageSpec(SEQUENCENUM, JdbcType.DECIMAL), + new PropertyStorageSpec(SOURCELSID, JdbcType.VARCHAR, 200), + new PropertyStorageSpec(_KEY, JdbcType.VARCHAR, 200), + new PropertyStorageSpec(QCSTATE, JdbcType.INTEGER), + new PropertyStorageSpec(PARTICIPANTSEQUENCENUM, JdbcType.VARCHAR, 200), + new PropertyStorageSpec(CREATED, JdbcType.TIMESTAMP), + new PropertyStorageSpec(MODIFIED, JdbcType.TIMESTAMP), + new PropertyStorageSpec(CREATED_BY, JdbcType.INTEGER), + new PropertyStorageSpec(MODIFIED_BY, JdbcType.INTEGER), + new PropertyStorageSpec(DATE, JdbcType.TIMESTAMP), + new PropertyStorageSpec(DataIntegrationService.Columns.TransformImportHash.getColumnName(), JdbcType.VARCHAR, 256) + )); + + + BASE_PROPERTIES = new HashSet<>(Arrays.asList( + new PropertyStorageSpec(DSROWID, JdbcType.BIGINT, 0, PropertyStorageSpec.Special.PrimaryKeyNonClustered, false, true, null), + new PropertyStorageSpec(PARTICIPANTID, JdbcType.VARCHAR, 32), + new PropertyStorageSpec(LSID, JdbcType.VARCHAR, 200), + new PropertyStorageSpec(SEQUENCENUM, JdbcType.DECIMAL), + new PropertyStorageSpec(SOURCELSID, JdbcType.VARCHAR, 200), + new PropertyStorageSpec(_KEY, JdbcType.VARCHAR, 200), + new PropertyStorageSpec(QCSTATE, JdbcType.INTEGER), + new PropertyStorageSpec(PARTICIPANTSEQUENCENUM, JdbcType.VARCHAR, 200), + new PropertyStorageSpec(CREATED, JdbcType.TIMESTAMP), + new PropertyStorageSpec(MODIFIED, JdbcType.TIMESTAMP), + new PropertyStorageSpec(CREATED_BY, JdbcType.INTEGER), + new PropertyStorageSpec(MODIFIED_BY, JdbcType.INTEGER), + new PropertyStorageSpec(DATE, JdbcType.TIMESTAMP), + new PropertyStorageSpec(DataIntegrationService.Columns.TransformImportHash.getColumnName(), JdbcType.VARCHAR, 256) + )); + + DATASPACE_PROPERTY_INDICES = new HashSet<>(Arrays.asList( + new PropertyStorageSpec.Index(false, true, CONTAINER, PARTICIPANTID, DATE), + new PropertyStorageSpec.Index(false, CONTAINER, QCSTATE), + new PropertyStorageSpec.Index(true, CONTAINER, PARTICIPANTID, SEQUENCENUM, _KEY), + new PropertyStorageSpec.Index(true, LSID), + new PropertyStorageSpec.Index(false, DATE) + )); + + PROPERTY_INDICES = new HashSet<>(Arrays.asList( + new PropertyStorageSpec.Index(false, true, PARTICIPANTID, DATE), + new PropertyStorageSpec.Index(false, QCSTATE), + new PropertyStorageSpec.Index(true, PARTICIPANTID, SEQUENCENUM, _KEY), + new PropertyStorageSpec.Index(true, LSID), + new PropertyStorageSpec.Index(false, DATE) + )); + } + + + protected DatasetDomainKind() + { + } + + @Override + abstract public String getKindName(); + + @Override + public Class getTypeClass() + { + return DatasetDomainKindProperties.class; + } + + @Override + public String getTypeLabel(Domain domain) + { + DatasetDefinition def = getDatasetDefinition(domain.getTypeURI()); + if (null == def) + return domain.getName(); + return def.getName(); + } + + @Override + public SQLFragment sqlObjectIdsInDomain(Domain domain) + { + DatasetDefinition def = getDatasetDefinition(domain.getTypeURI()); + if (null == def) + return new SQLFragment("NULL"); + TableInfo ti = def.getStorageTableInfo(false); + SQLFragment sql = new SQLFragment(); + sql.append("SELECT O.ObjectId FROM ").append(ti).append(" SD JOIN exp.Object O ON SD.Lsid=O.ObjectURI WHERE O.container=?"); + sql.add(def.getContainer()); + return sql; + } + + // Issue 16526: nobody should call this overload of generateDomainURI for DatasetDomainKind. Instead + // use the overload below with a unique id (the dataset's entityId). Assert is here to track down + // any callers. + // Lsid.toString() encodes incorrectly TODO: fix + @Override + public String generateDomainURI(String schemaName, String name, Container container, User user) + { + assert false; + return null; + } + + // Issue 16526: This specific generateDomainURI takes an id to uniquify the dataset. + public static String generateDomainURI(String name, String id, Container container) + { + String objectid = name == null ? "" : name; + if (null != id) + { + // normalize the object id + objectid += "-" + id.toLowerCase(); + } + return (new Lsid(LSID_PREFIX, "Folder-" + container.getRowId(), objectid)).toString(); + } + + @Override + public ActionURL urlShowData(Domain domain, ContainerUser containerUser) + { + Dataset def = getDatasetDefinition(domain.getTypeURI()); + ActionURL url = new ActionURL(StudyController.DatasetReportAction.class, containerUser.getContainer()); + url.addParameter("datasetId", "" + def.getDatasetId()); + return url; + } + + @Override + public ActionURL urlEditDefinition(Domain domain, ContainerUser containerUser) + { + Dataset def = getDatasetDefinition(domain.getTypeURI()); + ActionURL url = new ActionURL(StudyController.EditTypeAction.class, containerUser.getContainer()); + url.addParameter("datasetId", "" + def.getDatasetId()); + return url; + } + + @Override + public ActionURL urlCreateDefinition(String schemaName, String queryName, Container container, User user) + { + return new ActionURL(StudyController.DefineDatasetTypeAction.class, container); + } + + @Override + public boolean allowFileLinkProperties() + { + return true; + } + + @Override + public boolean allowMultiChoiceProperties() + { + return true; + } + + @Override + public boolean showDefaultValueSettings() + { + return true; + } + + @Override + public boolean allowUniqueConstraintProperties() + { + return true; + } + + @Override + public boolean allowCalculatedFields() + { + return true; + } + + DatasetDefinition getDatasetDefinition(String domainURI) + { + return StudyManager.getInstance().getDatasetDefinition(domainURI); + } + + // Issue 43898: Add the study subject name column to reserved fields + protected Set getStudySubjectReservedName(Domain domain) + { + HashSet fields = new CaseInsensitiveHashSet(); + if (null != domain) + { + Study study = StudyManager.getInstance().getStudy(domain.getContainer()); + if (null != study) + { + String participantIdField = study.getSubjectColumnName(); + fields.addAll(DomainUtil.getNameAndLabels(participantIdField)); + } + } + return Collections.unmodifiableSet(fields); + } + + @Override + public abstract @NotNull Set getReservedPropertyNames(Domain domain, User user); + + @Override + public Set getBaseProperties(Domain domain) + { + Set specs; + Study study = StudyManager.getInstance().getStudy(domain.getContainer()); + + if(study == null || study.isDataspaceStudy()) + { + specs = new HashSet<>(DATASPACE_BASE_PROPERTIES); + } + else + { + specs = new HashSet<>(BASE_PROPERTIES); + } + specs.addAll(super.getBaseProperties(domain)); + return specs; + } + + @Override + public Set getPropertyIndices(Domain domain) + { + Study study = StudyManager.getInstance().getStudy(domain.getContainer()); + + if(study == null || study.isDataspaceStudy()) + return DATASPACE_PROPERTY_INDICES; + + return PROPERTY_INDICES; + } + + @Override + public DbScope getScope() + { + return StudySchema.getInstance().getSchema().getScope(); + } + + @Override + public String getStorageSchemaName() + { + return StudySchema.getInstance().getDatasetSchemaName(); + } + + @Override + public void invalidate(Domain domain) + { + super.invalidate(domain); + DatasetDefinition def = getDatasetDefinition(domain.getTypeURI()); + if (null != def) + { + StudyManager.getInstance().uncache(def); + } + } + + @Override + public boolean canCreateDefinition(User user, Container container) + { + return container.hasPermission(user, AdminPermission.class); + } + + @Override + public Map processArguments(Container container, User user, Map arguments) + { + Map updatedArguments = new HashMap<>(arguments); + + // For backwards compatibility, map "demographics" => "demographicData" + if (arguments.containsKey("demographics")) + { + updatedArguments.put("demographicData", arguments.get("demographics")); + updatedArguments.remove("demographics"); + } + + // For backwards compatibility, map "categoryId" and "categoryName" => "category" + if (arguments.containsKey("categoryId")) + { + if (arguments.containsKey("categoryName")) + throw new IllegalArgumentException("Category ID and category name cannot both be specified."); + + ViewCategory category = ViewCategoryManager.getInstance().getCategory(container, asInteger(arguments.get("categoryId"))); + if (category == null) + throw new IllegalArgumentException("Unable to find a category with the ID : " + arguments.get("categoryId") + " in this folder."); + + updatedArguments.put("category", category.getLabel()); + updatedArguments.remove("categoryId"); + } + else if (arguments.containsKey("categoryName")) + { + updatedArguments.put("category", arguments.get("categoryName")); + updatedArguments.remove("categoryName"); + } + + return updatedArguments; + } + + @Nullable + @Override + public DatasetDomainKindProperties getDomainKindProperties(GWTDomain domain, Container container, User user) + { + Dataset ds = domain != null ? getDatasetDefinition(domain.getDomainURI()) : null; + return DatasetManager.get().getDatasetDomainKindProperties(container, ds != null ? ds.getDatasetId() : null); + } + + @Override + public Domain createDomain(GWTDomain domain, DatasetDomainKindProperties arguments, Container container, User user, + @Nullable TemplateInfo templateInfo, boolean forUpdate) + { + arguments.setName(StringUtils.trimToNull(domain.getName())); + String name = arguments.getName(); + if (name == null) + throw new IllegalArgumentException("Dataset name cannot be empty."); + String description = arguments.getDescription() != null ? arguments.getDescription() : domain.getDescription(); + String label = (arguments.getLabel() == null || arguments.getLabel().isEmpty()) ? arguments.getName() : arguments.getLabel(); + Integer cohortId = arguments.getCohortId(); + String tag = arguments.getTag(); + Integer datasetId = arguments.getDatasetId(); + String categoryName = arguments.getCategory(); + boolean demographics = arguments.isDemographicData(); + boolean isManagedField = arguments.isKeyPropertyManaged(); + String visitDatePropertyName = arguments.getVisitDatePropertyName(); + boolean useTimeKeyField = arguments.isUseTimeKeyField(); + boolean showByDefault = arguments.isShowByDefault(); + String dataSharing = arguments.getDataSharing(); + + // general dataset validation + validateDatasetProperties(arguments, container, user, domain, null); + + // create-case specific validation + StudyImpl study = StudyManager.getInstance().getStudy(container); + TimepointType timepointType = study.getTimepointType(); + if (timepointType.isVisitBased() && getKindName().equals(DateDatasetDomainKind.KIND_NAME)) + throw new IllegalArgumentException("Visit based studies require a visit based dataset domain. Please specify a kind name of : " + VisitDatasetDomainKind.KIND_NAME + "."); + else if (!timepointType.isVisitBased() && getKindName().equals(VisitDatasetDomainKind.KIND_NAME)) + throw new IllegalArgumentException("Date based studies require a date based dataset domain. Please specify a kind name of : " + DateDatasetDomainKind.KIND_NAME + "."); + if (timepointType.isVisitBased() && useTimeKeyField) + throw new IllegalArgumentException("Additional key property cannot be Time (from Date/Time) for visit based studies."); + + // Check for usage of Time as Key Field + String keyPropertyName = arguments.getKeyPropertyName(); + if (useTimeKeyField) + keyPropertyName = null; + + try (DbScope.Transaction transaction = ExperimentService.get().ensureTransaction()) + { + KeyManagementType managementType = KeyManagementType.None; + if (isManagedField) + { + String rangeUri = ""; + for (GWTPropertyDescriptor a : domain.getFields()) + { + if (keyPropertyName.equalsIgnoreCase(a.getName())) + { + rangeUri = a.getRangeURI(); + break; + } + + } + + PropertyDescriptor pd = new PropertyDescriptor(); + pd.setRangeURI(rangeUri); + managementType = KeyManagementType.getManagementTypeFromProp(pd.getPropertyType()); + } + + Integer categoryId = null; + ViewCategory category; + if (categoryName != null) + { + category = ViewCategoryManager.getInstance().getCategory(container, categoryName); + if (category != null) + { + categoryId = category.getRowId(); + } + else + { + String[] parts = ViewCategoryManager.getInstance().decode(categoryName); + category = ViewCategoryManager.getInstance().ensureViewCategory(container, user, parts); + categoryId = category.getRowId(); + } + } + + DatasetDefinition def = StudyPublishManager.getInstance().createDataset(user, new DatasetDefinition.Builder(name) + .setStudy(study) + .setKeyPropertyName(keyPropertyName) + .setDatasetId(datasetId) + .setDemographicData(demographics) + .setCategoryId(categoryId) + .setUseTimeKeyField(useTimeKeyField) + .setKeyManagementType(managementType) + .setShowByDefault(showByDefault) + .setLabel(label) + .setDescription(description) + .setCohortId(cohortId) + .setTag(tag) + .setVisitDatePropertyName(visitDatePropertyName) + .setDataSharing(dataSharing)); + + if (def.getDomain() != null) + { + List properties = domain.getFields(); + + Domain newDomain = def.getDomain(true); + if (newDomain != null) + { + Set reservedNames = getReservedPropertyNames(newDomain, user); + Set lowerReservedNames = reservedNames.stream().map(String::toLowerCase).collect(Collectors.toSet()); + Set existingProperties = newDomain.getProperties().stream().map(o -> o.getName().toLowerCase()).collect(Collectors.toSet()); + Map defaultValues = new HashMap<>(); + Set propertyUris = new CaseInsensitiveHashSet(); + + for (GWTPropertyDescriptor pd : properties) + { + if (lowerReservedNames.contains(pd.getName().toLowerCase()) || existingProperties.contains(pd.getName().toLowerCase())) + { + if (arguments.isStrictFieldValidation()) + throw new ApiUsageException("Property: " + pd.getName() + " is reserved or exists in the current domain."); + } + else + DomainUtil.addProperty(newDomain, pd, defaultValues, propertyUris, null); + } + + newDomain.save(user, arguments.getAuditRecordMap(), domain.getCalculatedFields()); + + List indices = domain.getIndices(); + newDomain.setPropertyIndices(indices, lowerReservedNames); + StorageProvisioner.get().ensureTableIndices(newDomain); + + QueryService.get().saveCalculatedFieldsMetadata("study", name, null, domain.getCalculatedFields(), false, user, container); + } + else + throw new IllegalArgumentException("Failed to create domain for dataset : " + name + "."); + } + + transaction.commit(); + return study.getDataset(def.getDatasetId()).getDomain(forUpdate); + } + catch (Exception e) + { + UnexpectedException.rethrow(e); // don't re-wrap runtime exceptions + return null; // can't get here + } + } + + private void validateDatasetProperties(DatasetDomainKindProperties datasetProperties, Container container, User user, GWTDomain domain, DatasetDefinition def) + { + String name = StringUtils.trimToEmpty(datasetProperties.getName()); + String keyPropertyName = datasetProperties.getKeyPropertyName(); + Integer datasetId = datasetProperties.getDatasetId(); + boolean isManagedField = datasetProperties.isKeyPropertyManaged(); + boolean useTimeKeyField = datasetProperties.isUseTimeKeyField(); + boolean isDemographicData = datasetProperties.isDemographicData(); + + if (!container.hasPermission(user, AdminPermission.class)) + throw new IllegalArgumentException("You do not have permissions to edit dataset definitions in this container."); + + Study study = StudyService.get().getStudy(container); + if (study == null) + throw new IllegalArgumentException("A study does not exist for this container."); + + // Name related exceptions + + if (name == null || name.isEmpty()) + throw new IllegalArgumentException("Dataset name cannot be empty."); + + if (name.length() > 200) + throw new IllegalArgumentException("Dataset name must be under 200 characters."); + + if (!name.equals(domain.getName()) || def == null) + { + // issue 17766: check if dataset or query exist with this name + if (null != StudyManager.getInstance().getDatasetDefinitionByName(study, name) || null != QueryService.get().getQueryDef(user, container, "study", name)) + throw new IllegalArgumentException("A Dataset or Query already exists with the name \"" + name + "\"."); + + StudyQuerySchema schema = StudyQuerySchema.createSchema(StudyManager.getInstance().getStudy(container), User.getSearchUser()); + + // Now check standard study tables + if (schema.getTableNames().contains(name)) + throw new IllegalArgumentException("A study table exists with the name \"" + name + "\"."); + + String datasetNameError = DomainUtil.validateDomainName(name, getKindName(), false); + if (datasetNameError != null) + throw new IllegalArgumentException(datasetNameError); + } + + // Label related exceptions + + String label = Objects.requireNonNullElse(datasetProperties.getLabel(), name); // Need to verify label uniqueness even if label is unspecified + + if ((def == null || !def.getLabel().equals(label)) && null != StudyManager.getInstance().getDatasetDefinitionByLabel(study, label)) + throw new IllegalArgumentException("A Dataset already exists with the label \"" + label +"\"."); + + // Additional key related exceptions + + if ("".equals(keyPropertyName)) + throw new IllegalArgumentException("Please select a field name for the additional key."); + + if (isManagedField && (keyPropertyName == null || keyPropertyName.isEmpty())) + throw new IllegalArgumentException("Additional Key Column name must be specified if field is managed."); + + if (useTimeKeyField && isManagedField) + throw new IllegalArgumentException("Additional key cannot be a managed field if KeyPropertyName is Time (from Date/Time)."); + + if (useTimeKeyField && !(keyPropertyName == null || keyPropertyName.equals(TIME_KEY_FIELD_KEY))) + throw new IllegalArgumentException("KeyPropertyName should not be provided when using additional key of Time (from Date/Time)."); + + if (isDemographicData && (isManagedField || keyPropertyName != null)) + throw new IllegalArgumentException("There cannot be an Additional Key Column if the dataset is Demographic Data."); + + if (!useTimeKeyField && null != keyPropertyName && null == domain.getFieldByName(keyPropertyName)) + throw new IllegalArgumentException("Additional Key Column name \"" + keyPropertyName +"\" must be the name of a column."); + + if (null != keyPropertyName && isManagedField) + { + String rangeURI = domain.getFieldByName(keyPropertyName).getRangeURI(); + if (!(rangeURI.endsWith("int") || rangeURI.endsWith("double") || rangeURI.endsWith("string"))) + throw new IllegalArgumentException("If Additional Key Column is managed, the column type must be numeric or text-based."); + } + + // Other exception(s) + + if (null != datasetId && (null == def || def.getDatasetId() != datasetId) && null != study.getDataset(datasetId)) + throw new IllegalArgumentException("A Dataset already exists with the datasetId \"" + datasetId +"\"."); + + if (!study.getShareVisitDefinitions() && null != datasetProperties.getDataSharing() && !datasetProperties.getDataSharing().equals("NONE")) + throw new IllegalArgumentException("Illegal value set for data sharing option."); + } + + private void checkCanUpdate(DatasetDefinition def, Container container, User user, DatasetDomainKindProperties datasetProperties, + GWTDomain original, GWTDomain update) + { + if (null == def) + throw new IllegalArgumentException("Dataset not found."); + + if (!def.canUpdateDefinition(user)) + throw new IllegalArgumentException("Shared dataset can not be edited in this folder."); + + if (datasetProperties.getLabel() == null || datasetProperties.getLabel().isEmpty()) + throw new IllegalArgumentException("Dataset label cannot be empty."); + + if (null == PropertyService.get().getDomain(container, update.getDomainURI())) + throw new IllegalArgumentException("Domain not found: " + update.getDomainURI() + "."); + + if (!def.getTypeURI().equals(original.getDomainURI()) || !def.getTypeURI().equals(update.getDomainURI())) + throw new IllegalArgumentException("Illegal Argument"); + + if (datasetProperties.isDemographicData() && !def.isDemographicData() && !StudyManager.getInstance().isDataUniquePerParticipant(def)) + { + String noun = StudyService.get().getSubjectNounSingular(container); + throw new IllegalArgumentException("This dataset currently contains more than one row of data per " + noun + + ". Demographic data includes one row of data per " + noun + "."); + } + } + + private @NotNull ValidationException updateDomainDescriptor(GWTDomain original, GWTDomain update, + @Nullable DatasetDefinition oldDef, @Nullable DatasetDomainKindProperties datasetPropertiesUpdate, Container container, User user, String userComment) + { + StringBuilder changeDetails = new StringBuilder(); + boolean hasNameChange = false; + Map oldProps = null; + Map newProps = datasetPropertiesUpdate != null ? datasetPropertiesUpdate.getAuditRecordMap() : null; + + if (oldDef != null) + { + oldProps = oldDef.getAuditRecordMap(); + if (newProps == null) + newProps = oldProps; // no update + + hasNameChange = !datasetPropertiesUpdate.getName().equals(oldDef.getName()); + if (hasNameChange) + changeDetails.append("The name of the dataset '" + oldDef.getName() + "' was changed to '" + datasetPropertiesUpdate.getName() + "'."); + } + + ValidationException exception = new ValidationException(); + exception.addErrors(DomainUtil.updateDomainDescriptor(original, update, container, user, hasNameChange, changeDetails.toString(), userComment, oldProps, newProps)); + return exception; + } + + private ValidationException updateDataset(DatasetDomainKindProperties datasetProperties, String domainURI, ValidationException exception, + StudyImpl study, Container container, User user, DatasetDefinition def) + { + try + { + // Check for usage of Time as Key Field + boolean useTimeKeyField = datasetProperties.isUseTimeKeyField(); + if (useTimeKeyField) + datasetProperties.setKeyPropertyName(null); + + // Default is no key management + KeyManagementType keyType = KeyManagementType.None; + String keyPropertyName = datasetProperties.getKeyPropertyName(); + if (datasetProperties.getKeyPropertyName() != null) + { + Domain domain = PropertyService.get().getDomain(container, domainURI); + for (DomainProperty dp : domain.getProperties()) + { + if (dp.getName().equalsIgnoreCase(datasetProperties.getKeyPropertyName())) + { + keyPropertyName = dp.getName(); + + // Be sure that the user really wants a managed key, not just that disabled select box still had a value + if (datasetProperties.isKeyPropertyManaged()) + keyType = KeyManagementType.getManagementTypeFromProp(dp.getPropertyDescriptor().getPropertyType()); + + break; + } + } + } + + DatasetDefinition updated = def.createMutable(); + + // Clear the category ID so that it gets regenerated based on the new string - see issue 19649 + updated.setCategoryId(null); + + BeanUtils.copyProperties(updated, datasetProperties); + updated.setKeyPropertyName(keyPropertyName); + updated.setKeyManagementType(keyType); + updated.setUseTimeKeyField(useTimeKeyField); + + List errors = new ArrayList<>(); + StudyManager.getInstance().updateDatasetDefinition(user, updated, errors); + StudyManager.datasetModified(updated, true); + + for (String errorMsg: errors) + exception.addGlobalError(errorMsg); + return exception; + } + catch (RuntimeSQLException e) + { + return exception.addGlobalError("Additional key column must have unique values."); + } + catch (Exception e) + { + throw new RuntimeException(e); + } + } + + @Override + public @NotNull ValidationException updateDomain(GWTDomain original, GWTDomain update, + @Nullable DatasetDomainKindProperties datasetProperties, Container container, User user, boolean includeWarnings, String userComment) + { + assert original.getDomainURI().equals(update.getDomainURI()); + StudyImpl study = StudyManager.getInstance().getStudy(container); + DatasetDefinition def = null; + boolean hasNameChange = false; + + if (datasetProperties != null) + { + def = study.getDataset(datasetProperties.getDatasetId()); + validateDatasetProperties(datasetProperties, container, user, update, def); + checkCanUpdate(def, container, user, datasetProperties, original, update); + String updatedName = StringUtils.trimToEmpty(datasetProperties.getName()); + datasetProperties.setName(updatedName); + hasNameChange = !def.getName().equals(updatedName); + } + + // Acquire lock before we actually start the transaction to avoid deadlocks when it's refreshed during the process + Lock[] locks = def == null ? new Lock[0] : new Lock[] { def.getDomainLoadingLock() }; + try (DbScope.Transaction transaction = StudySchema.getInstance().getScope().ensureTransaction(locks)) + { + ValidationException exception = updateDomainDescriptor(original, update, def, datasetProperties, container, user, userComment); + + QueryService.get().saveCalculatedFieldsMetadata("study", update.getQueryName(), hasNameChange ? datasetProperties.getName() : null, update.getCalculatedFields(), !original.getCalculatedFields().isEmpty(), user, container); + + if (!exception.hasErrors() && def != null) + exception = updateDataset(datasetProperties, original.getDomainURI(), exception, study, container, user, def); + + if (!exception.hasErrors()) + transaction.commit(); + + return exception; + } + catch (MetadataUnavailableException e) + { + return new ValidationException(e.getMessage()); + } + finally + { + if (def != null) + StudyManager.getInstance().uncache(def); + } + } + + @Override + public void deleteDomain(User user, Domain domain, @Nullable String auditUserComment) + { + DatasetDefinition def = StudyManager.getInstance().getDatasetDefinition(domain.getTypeURI()); + if (def == null) + throw new NotFoundException("Dataset not found: " + domain.getTypeURI()); + + StudyImpl study = StudyManager.getInstance().getStudy(domain.getContainer()); + if (study == null) + throw new IllegalArgumentException("A study does not exist for this folder"); + + try (DbScope.Transaction transaction = StudySchema.getInstance().getSchema().getScope().ensureTransaction()) + { + StudyManager.getInstance().deleteDataset(study, user, def, false, auditUserComment); + transaction.commit(); + } + } + + @Override + public boolean isDeleteAllDataOnFieldImport() + { + return true; + } + + + @Override + public TableInfo getTableInfo(User user, Container container, String name, @Nullable ContainerFilter cf) + { + StudyImpl study = StudyManager.getInstance().getStudy(container); + if (null == study) + return null; + StudyQuerySchema schema = StudyQuerySchema.createSchema(study, user); + DatasetDefinition dsd = schema.getDatasetDefinitionByName(name); + if (null == dsd) + return null; + + return schema.getTable(name, cf, true, false); + } + + @Override + public void afterLoadTable(SchemaTableInfo ti, Domain domain) + { + // Grab the "standard" properties and apply them to this dataset table + TableInfo template = DatasetDefinition.getTemplateTableInfo(); + + for (PropertyStorageSpec pss : domain.getDomainKind().getBaseProperties(domain)) + { + ColumnInfo c = ti.getColumn(pss.getName()); + ColumnInfo tCol = template.getColumn(pss.getName()); + // The column may be null if the dataset is being deleted in the background + if (null != tCol && c != null) + { + ((BaseColumnInfo)c).setExtraAttributesFrom(tCol); + + // When copying a column, the hidden bit is not propagated, so we need to do it manually + if (tCol.isHidden()) + ((BaseColumnInfo)c).setHidden(true); + } + } + } + + @Override + public boolean supportsPhiLevel() + { + return ComplianceService.get().isComplianceSupported(); + } + + // Query datasets don't have a domain kind, so check here if the dataset is a query dataset + @Override + public boolean isProvisioned(Container container, String name) + { + return super.isProvisioned(container, name) && !isQueryDataset(container, name); + } + + public static boolean isQueryDataset(Container container, String queryName) + { + StudyService ss = StudyService.get(); + if (ss == null) + return false; + + Dataset dt = ss.getDataset(container, ss.getDatasetIdByName(container, queryName)); + if (dt == null) + return false; + + return dt.isQueryDataset(); + } +} From 65538dc9a7b1769c599809788b1b87042dd1887d Mon Sep 17 00:00:00 2001 From: XingY Date: Mon, 12 Jan 2026 19:09:30 -0800 Subject: [PATCH 5/6] support filters --- api/schemas/queryCustomView.xsd | 1 + api/src/org/labkey/api/data/CompareType.java | 343 ++++++++++++++++++ .../org/labkey/query/QueryServiceImpl.java | 5 + 3 files changed, 349 insertions(+) diff --git a/api/schemas/queryCustomView.xsd b/api/schemas/queryCustomView.xsd index 61983586694..3c0ce4e96ec 100644 --- a/api/schemas/queryCustomView.xsd +++ b/api/schemas/queryCustomView.xsd @@ -96,6 +96,7 @@ + diff --git a/api/src/org/labkey/api/data/CompareType.java b/api/src/org/labkey/api/data/CompareType.java index 8d20eb3a243..b8f1961f0d5 100644 --- a/api/src/org/labkey/api/data/CompareType.java +++ b/api/src/org/labkey/api/data/CompareType.java @@ -808,6 +808,349 @@ public QClause createFilterClause(@NotNull FieldKey fieldKey, Object value) } }; + public Collection getCollectionParam(Object value) + { + if (value instanceof Collection) + { + return (Collection)value; + } + else + { + List values = new ArrayList<>(); + if (value != null) + { + if (value.toString().trim().isEmpty()) + { + values.add(null); + } + else + { + values.addAll(parseParams(value, getValueSeparator(), isNewLineSeparatorAllowed())); + } + } + return values; + } + } + /** TODO: + * + * + */ + public static final CompareType ARRAY_CONTAINS_ALL = new CompareType("Contains All", "arraycontainsall", "ARRAYCONTAINSALL", true, null, OperatorType.ARRAYCONTAINSALL) + { + @Override + public ArrayContainsAllClause createFilterClause(@NotNull FieldKey fieldKey, Object value) + { + return new ArrayContainsAllClause(fieldKey, getCollectionParam(value)); + } + + @Override + public boolean meetsCriteria(ColumnRenderProperties col, Object value, Object[] filterValues) + { + throw new UnsupportedOperationException("Conditional formatting not yet supported for Multi Choices"); + } + + @Override + public String getValueSeparator() + { + return ArrayClause.ARRAY_VALUE_SEPARATOR; + } + }; + + public static final CompareType ARRAY_CONTAINS_ANY = new CompareType("Contains Any", "arraycontainsany", "ARRAYCONTAINSANY", true, null, OperatorType.ARRAYCONTAINSANY) + { + @Override + public ArrayContainsAnyClause createFilterClause(@NotNull FieldKey fieldKey, Object value) + { + return new ArrayContainsAnyClause(fieldKey, getCollectionParam(value)); + } + + @Override + public boolean meetsCriteria(ColumnRenderProperties col, Object value, Object[] filterValues) + { + throw new UnsupportedOperationException("Conditional formatting not yet supported for Multi Choices"); + } + + @Override + public String getValueSeparator() + { + return ArrayClause.ARRAY_VALUE_SEPARATOR; + } + }; + + public static final CompareType ARRAY_CONTAINS_NONE = new CompareType("Contains None", "arraycontainsnone", "ARRAYCONTAINSNONE", true, null, OperatorType.ARRAYCONTAINSNONE) + { + @Override + public ArrayContainsNoneClause createFilterClause(@NotNull FieldKey fieldKey, Object value) + { + return new ArrayContainsNoneClause(fieldKey, getCollectionParam(value)); + } + + @Override + public boolean meetsCriteria(ColumnRenderProperties col, Object value, Object[] filterValues) + { + throw new UnsupportedOperationException("Conditional formatting not yet supported for Multi Choices"); + } + + @Override + public String getValueSeparator() + { + return ArrayClause.ARRAY_VALUE_SEPARATOR; + } + }; + + public static final CompareType ARRAY_MATCHES = new CompareType("Contains Exactly", "arraymatches", "ARRAYMATCHES", true, null, OperatorType.ARRAYMATCHES) + { + @Override + public ArrayMatchesClause createFilterClause(@NotNull FieldKey fieldKey, Object value) + { + return new ArrayMatchesClause(fieldKey, getCollectionParam(value)); + } + + @Override + public boolean meetsCriteria(ColumnRenderProperties col, Object value, Object[] filterValues) + { + throw new UnsupportedOperationException("Conditional formatting not yet supported for Multi Choices"); + } + + @Override + public String getValueSeparator() + { + return ArrayClause.ARRAY_VALUE_SEPARATOR; + } + }; + + public static final CompareType ARRAY_NOT_MATCHES = new CompareType("Does Not Contain Exactly", "arraynotmatches", "ARRAYNOTMATCHES", true, null, OperatorType.ARRAYNOTMATCHES) + { + @Override + public ArrayNotMatchesClause createFilterClause(@NotNull FieldKey fieldKey, Object value) + { + return new ArrayNotMatchesClause(fieldKey, getCollectionParam(value)); + } + + @Override + public boolean meetsCriteria(ColumnRenderProperties col, Object value, Object[] filterValues) + { + throw new UnsupportedOperationException("Conditional formatting not yet supported for Multi Choices"); + } + + @Override + public String getValueSeparator() + { + return ArrayClause.ARRAY_VALUE_SEPARATOR; + } + }; + + public static abstract class ArrayClause extends SimpleFilter.MultiValuedFilterClause + { + public static final String ARRAY_VALUE_SEPARATOR = ","; + + public ArrayClause(@NotNull FieldKey fieldKey, CompareType comparison, Collection params, boolean negated) + { + super(fieldKey, comparison, params, negated); + } + + public SQLFragment[] getParamSQLFragments(SqlDialect dialect) + { + Object[] params = getParamVals(); + SQLFragment[] fragments = new SQLFragment[params.length]; + + JdbcType type = null; + // Try to infer the type from the first non-null parameter + for (Object param : params) + { + if (param != null) + { + type = JdbcType.valueOf(param.getClass()); + break; + } + } + + for (int i = 0; i < params.length; i++) + fragments[i] = new SQLFragment().append(escapeLabKeySqlValue(params[i], type)); + + return fragments; + } + + public Pair getSqlFragments(Map columnMap, SqlDialect dialect) + { + SQLFragment[] paramValues = getParamSQLFragments(dialect); + + if (paramValues == null || paramValues.length == 0) + return null; + + ColumnInfo colInfo = columnMap != null ? columnMap.get(_fieldKey) : null; + var alias = SimpleFilter.getAliasForColumnFilter(dialect, colInfo, _fieldKey); + + SQLFragment valuesFragment = dialect.array_construct(paramValues); + SQLFragment columnFragment = new SQLFragment().appendIdentifier(alias); + + return new Pair<>(valuesFragment, columnFragment); + } + + } + + private static class ArrayContainsAllClause extends ArrayClause + { + + public ArrayContainsAllClause(@NotNull FieldKey fieldKey, Collection params) + { + super(fieldKey, CompareType.ARRAY_CONTAINS_ALL, params, false); + } + + @Override + public SQLFragment toSQLFragment(Map columnMap, SqlDialect dialect) + { + Pair valueFieldSql = getSqlFragments(columnMap, dialect); + if (valueFieldSql == null) + return new SQLFragment("1=2"); + + return dialect.array_all_in_array(valueFieldSql.first, valueFieldSql.second); + } + + @Override + public String getLabKeySQLWhereClause(Map columnMap) + { + return "array_contains_all(" + getLabKeySQLColName(_fieldKey) + ", " + getParamVals()[0] + ")"; + } + + @Override + public void appendFilterText(StringBuilder sb, ColumnNameFormatter formatter) + { + Object[] params = getParamVals(); + sb.append("contains all of ").append(Arrays.toString(params)); + } + + } + + private static class ArrayContainsAnyClause extends ArrayClause + { + + public ArrayContainsAnyClause(@NotNull FieldKey fieldKey, CompareType comparison, Collection params, boolean negated) + { + super(fieldKey, comparison, params, negated); + } + + public ArrayContainsAnyClause(@NotNull FieldKey fieldKey, Collection params) + { + this(fieldKey, CompareType.ARRAY_CONTAINS_ANY, params, false); + } + + @Override + public SQLFragment toSQLFragment(Map columnMap, SqlDialect dialect) + { + Pair valueFieldSql = getSqlFragments(columnMap, dialect); + if (valueFieldSql == null) + return new SQLFragment("1=2"); + + SQLFragment sql = dialect.array_some_in_array(valueFieldSql.first, valueFieldSql.second); + if (!_negated) + return sql; + return new SQLFragment(" NOT (").append(sql).append(")"); + } + + @Override + public String getLabKeySQLWhereClause(Map columnMap) + { + return "array_contains_any(" + getLabKeySQLColName(_fieldKey) + ", " + getParamVals()[0] + ")"; + } + + @Override + public void appendFilterText(StringBuilder sb, ColumnNameFormatter formatter) + { + Object[] params = getParamVals(); + sb.append("contains at least one of ").append(Arrays.toString(params)); + } + + } + + private static class ArrayContainsNoneClause extends ArrayContainsAnyClause + { + + public ArrayContainsNoneClause(@NotNull FieldKey fieldKey, Collection params) + { + super(fieldKey, CompareType.ARRAY_CONTAINS_NONE, params, true); + } + + @Override + public String getLabKeySQLWhereClause(Map columnMap) + { + return "array_contains_none(" + getLabKeySQLColName(_fieldKey) + ", " + getParamVals()[0] + ")"; + } + + @Override + public void appendFilterText(StringBuilder sb, ColumnNameFormatter formatter) + { + Object[] params = getParamVals(); + sb.append("contains none of ").append(Arrays.toString(params)); + } + + } + + private static class ArrayMatchesClause extends ArrayClause + { + + public ArrayMatchesClause(@NotNull FieldKey fieldKey, CompareType comparison, Collection params, boolean negated) + { + super(fieldKey, comparison, params, negated); + } + + public ArrayMatchesClause(@NotNull FieldKey fieldKey, Collection params) + { + this(fieldKey, CompareType.ARRAY_MATCHES, params, false); + } + + @Override + public SQLFragment toSQLFragment(Map columnMap, SqlDialect dialect) + { + Pair valueFieldSql = getSqlFragments(columnMap, dialect); + if (valueFieldSql == null) + return new SQLFragment("1=2"); + + SQLFragment sql = dialect.array_same_array(valueFieldSql.first, valueFieldSql.second); + if (!_negated) + return sql; + return new SQLFragment(" NOT (").append(sql).append(")"); + } + + @Override + public String getLabKeySQLWhereClause(Map columnMap) + { + return "array_is_same(" + getLabKeySQLColName(_fieldKey) + ", " + getParamVals()[0] + ")"; + } + + @Override + public void appendFilterText(StringBuilder sb, ColumnNameFormatter formatter) + { + Object[] params = getParamVals(); + sb.append("contains the same elements as ").append(Arrays.toString(params)); + } + + } + + private static class ArrayNotMatchesClause extends ArrayMatchesClause + { + + public ArrayNotMatchesClause(@NotNull FieldKey fieldKey, Collection params) + { + super(fieldKey, CompareType.ARRAY_NOT_MATCHES, params, true); + } + + @Override + public String getLabKeySQLWhereClause(Map columnMap) + { + return "NOT array_is_same(" + getLabKeySQLColName(_fieldKey) + ", " + getParamVals()[0] + ")"; + } + + @Override + public void appendFilterText(StringBuilder sb, ColumnNameFormatter formatter) + { + Object[] params = getParamVals(); + sb.append("does not contain the same elements as ").append(Arrays.toString(params)); + } + + } + + private static class QClause extends CompareType.CompareClause { private List _queryColumns = null; diff --git a/query/src/org/labkey/query/QueryServiceImpl.java b/query/src/org/labkey/query/QueryServiceImpl.java index 45749c18d19..d44943d4f2a 100644 --- a/query/src/org/labkey/query/QueryServiceImpl.java +++ b/query/src/org/labkey/query/QueryServiceImpl.java @@ -309,6 +309,11 @@ public void moduleChanged(Module module) CompareType.NONBLANK, CompareType.MV_INDICATOR, CompareType.NO_MV_INDICATOR, + CompareType.ARRAY_CONTAINS_ALL, + CompareType.ARRAY_CONTAINS_ANY, + CompareType.ARRAY_CONTAINS_NONE, + CompareType.ARRAY_MATCHES, + CompareType.ARRAY_NOT_MATCHES, CompareType.Q, WHERE, INDESCENDANTSOF, From 9801a9cf2f80e96e76f7c9f406b9d27c811c7fba Mon Sep 17 00:00:00 2001 From: XingY Date: Tue, 13 Jan 2026 12:35:24 -0800 Subject: [PATCH 6/6] clean --- api/src/org/labkey/api/data/ColumnRenderPropertiesImpl.java | 1 - api/src/org/labkey/api/data/CompareType.java | 3 ++- api/src/org/labkey/api/data/dialect/SqlDialect.java | 2 +- api/src/org/labkey/api/exp/PropertyType.java | 2 ++ experiment/src/org/labkey/experiment/ExperimentModule.java | 2 +- 5 files changed, 6 insertions(+), 4 deletions(-) diff --git a/api/src/org/labkey/api/data/ColumnRenderPropertiesImpl.java b/api/src/org/labkey/api/data/ColumnRenderPropertiesImpl.java index 1025106138f..06c4dcd35f8 100644 --- a/api/src/org/labkey/api/data/ColumnRenderPropertiesImpl.java +++ b/api/src/org/labkey/api/data/ColumnRenderPropertiesImpl.java @@ -48,7 +48,6 @@ public abstract class ColumnRenderPropertiesImpl implements MutableColumnRenderP public static final String STORAGE_UNIQUE_ID_CONCEPT_URI = "http://www.labkey.org/types#storageUniqueId"; public static final String STORAGE_UNIQUE_ID_SEQUENCE_PREFIX = "org.labkey.api.StorageUniqueId"; public static final String TEXT_CHOICE_CONCEPT_URI = "http://www.labkey.org/types#textChoice"; - //public static final String MULTI_VALUE_TEXT_CHOICE_CONCEPT_URI = "http://www.labkey.org/types#mvTextChoice"; public static final String NON_NEGATIVE_NUMBER_CONCEPT_URI = "http://www.labkey.org/types#nonNegativeNumber"; protected SortDirection _sortDirection = SortDirection.ASC; diff --git a/api/src/org/labkey/api/data/CompareType.java b/api/src/org/labkey/api/data/CompareType.java index b8f1961f0d5..52760c53100 100644 --- a/api/src/org/labkey/api/data/CompareType.java +++ b/api/src/org/labkey/api/data/CompareType.java @@ -808,7 +808,7 @@ public QClause createFilterClause(@NotNull FieldKey fieldKey, Object value) } }; - public Collection getCollectionParam(Object value) + protected Collection getCollectionParam(Object value) { if (value instanceof Collection) { @@ -831,6 +831,7 @@ public Collection getCollectionParam(Object value) return values; } } + /** TODO: * * diff --git a/api/src/org/labkey/api/data/dialect/SqlDialect.java b/api/src/org/labkey/api/data/dialect/SqlDialect.java index c68b4268bc4..85cc60ec933 100644 --- a/api/src/org/labkey/api/data/dialect/SqlDialect.java +++ b/api/src/org/labkey/api/data/dialect/SqlDialect.java @@ -1303,7 +1303,7 @@ public String getJDBCArrayType(Object object) public String getJDBCArrayType(Object[] array) { String typeName; - if (array.length == 0) + if (array.length == 0 || array[0] == null) { // Handle empty arrays by inferring the SQL element type from the Java component type. // Primary target is String[0], but handle a reasonable set of common types defensively. diff --git a/api/src/org/labkey/api/exp/PropertyType.java b/api/src/org/labkey/api/exp/PropertyType.java index 50840fcda29..fa55721091f 100644 --- a/api/src/org/labkey/api/exp/PropertyType.java +++ b/api/src/org/labkey/api/exp/PropertyType.java @@ -240,6 +240,8 @@ protected void setValue(ObjectProperty property, Object value) property.arrayValue = MultiChoice.Array.from(array); else if (value != null) property.arrayValue = MultiChoice.Array.from(new Object[]{value}); + else + property.arrayValue = null; } @Override diff --git a/experiment/src/org/labkey/experiment/ExperimentModule.java b/experiment/src/org/labkey/experiment/ExperimentModule.java index 4f70bc8b967..e94b8ee1c15 100644 --- a/experiment/src/org/labkey/experiment/ExperimentModule.java +++ b/experiment/src/org/labkey/experiment/ExperimentModule.java @@ -270,7 +270,7 @@ protected void init() OptionalFeatureService.get().addExperimentalFeatureFlag(NameGenerator.EXPERIMENTAL_ALLOW_GAP_COUNTER, "Allow gap with withCounter and rootSampleCount expression", "Check this option if gaps in the count generated by withCounter or rootSampleCount name expression are allowed.", true); OptionalFeatureService.get().addExperimentalFeatureFlag(AppProps.MULTI_VALUE_TEXT_CHOICE, "Allow multi-value Text Choice properties", - "Support selecting more than one values for text choice fields", false); + "Support selecting more than one value for text choice fields", false); } OptionalFeatureService.get().addExperimentalFeatureFlag(AppProps.QUANTITY_COLUMN_SUFFIX_TESTING, "Quantity column suffix testing", "If a column name contains a \"__\" suffix, this feature allows for testing it as a Quantity display column", false);