diff --git a/api/src/org/labkey/api/data/MaterializedQueryHelper.java b/api/src/org/labkey/api/data/MaterializedQueryHelper.java index 4750e55189a..425e063787d 100644 --- a/api/src/org/labkey/api/data/MaterializedQueryHelper.java +++ b/api/src/org/labkey/api/data/MaterializedQueryHelper.java @@ -56,12 +56,14 @@ private class Materialized private final long _created; private final String _cacheKey; private final String _fromSql; + private final String _tableName; private final ArrayList _invalidators = new ArrayList<>(3); - Materialized(String cacheKey, long created, String sql) + Materialized(String tableName, String cacheKey, long created, String sql) { _created = created; _cacheKey = cacheKey; + _tableName = tableName; _fromSql = sql; } @@ -191,11 +193,11 @@ private String makeKey(DbScope.Transaction t, Container c) private final String _prefix; private final DbScope _scope; private final SQLFragment _selectQuery; + private final boolean _isSelectIntoSql; private final SQLFragment _uptodateQuery; private final Supplier _supplier; private final List _indexes = new ArrayList<>(); private final long _maxTimeToCache; - private final boolean _perContainer; private final LinkedHashMap _map = new LinkedHashMap() { @Override @@ -214,7 +216,7 @@ protected boolean removeEldestEntry(Map.Entry eldest) private boolean _closed = false; private MaterializedQueryHelper(String prefix, DbScope scope, SQLFragment select, @Nullable SQLFragment uptodate, Supplier supplier, @Nullable Collection indexes, long maxTimeToCache, - boolean perContainer) + boolean isSelectIntoSql) { _prefix = StringUtils.defaultString(prefix,"mat"); _scope = scope; @@ -222,11 +224,9 @@ private MaterializedQueryHelper(String prefix, DbScope scope, SQLFragment select _uptodateQuery = uptodate; _supplier = supplier; _maxTimeToCache = maxTimeToCache; - _perContainer = perContainer; if (null != indexes) _indexes.addAll(indexes); - if (perContainer) - throw new UnsupportedOperationException("NYI"); + _isSelectIntoSql = isSelectIntoSql; assert MemTracker.get().put(this); } @@ -303,57 +303,50 @@ public SQLFragment getFromSql(String tableAlias, Container c) { if (null == _selectQuery) throw new IllegalStateException("Must specify source query in constructor or in getFromSql()"); - return getFromSql(_selectQuery, tableAlias, c); + return getFromSql(_selectQuery, _isSelectIntoSql, tableAlias, c); } - /* NOTE: we do not want to hold synchronized(this) while doing any SQL operations */ - public SQLFragment getFromSql(@NotNull SQLFragment selectQuery, String tableAlias, Container c) + + public boolean isCached(Container c) { - Materialized materialized = null; - final String txCacheKey = makeKey(_scope.getCurrentTransaction(), c); - final long now = HeartBeat.currentTimeMillis(); + if (null == _selectQuery) + throw new IllegalStateException("Must specify source query in constructor or in getFromSql()"); + return null != getMaterialized(makeKey(_scope.getCurrentTransaction(), c)); + } - synchronized (this) - { - if (_closed) - throw new IllegalStateException(); - if (null != c) - throw new UnsupportedOperationException(); - _countGetFromSql.incrementAndGet(); + public void upsert(SQLFragment sqlf) + { + String txCacheKey = makeKey(_scope.getCurrentTransaction(), null); + Materialized m = getMaterialized(txCacheKey); + if (null == m) + return; + String sql = sqlf.getSQL().replace("${NAME}", m._tableName); + List params = sqlf.getParams(); + new SqlExecutor(_scope).execute(new SQLFragment(sql,params)); + } - if (_scope.isTransactionActive()) - materialized = _map.get(txCacheKey); - if (null == materialized) - materialized = _map.get(makeKey(null, c)); - } + /* used by FLow directly for some reason */ + public SQLFragment getFromSql(@NotNull SQLFragment selectQuery, String tableAlias, Container c) + { + return getFromSql(selectQuery, false, tableAlias, c); + } - if (null != materialized) - { - boolean replace = false; - for (Invalidator i : materialized._invalidators) - { - CacheCheck cc = i.checkValid(materialized._created); - if (cc != CacheCheck.OK) - replace = true; - } - if (replace) - { - synchronized (this) - { - _map.remove(materialized._cacheKey); - materialized = null; - } - } - } + /* NOTE: we do not want to hold synchronized(this) while doing any SQL operations */ + public SQLFragment getFromSql(@NotNull SQLFragment selectQuery, boolean isSelectInto, String tableAlias, Container c) + { + final String txCacheKey = makeKey(_scope.getCurrentTransaction(), c); + final long now = HeartBeat.currentTimeMillis(); + + Materialized materialized = getMaterialized(txCacheKey); if (null == materialized) { _countSelectInto.incrementAndGet(); DbSchema temp = DbSchema.getTemp(); String name = _prefix + "_" + GUID.makeHash(); - materialized = new Materialized(txCacheKey, now, "\"" + temp.getName() + "\".\"" + name + "\""); + materialized = new Materialized(name, txCacheKey, now, "\"" + temp.getName() + "\".\"" + name + "\""); materialized.addMaxTimeToCache(_maxTimeToCache); materialized.addUpToDateQuery(_uptodateQuery); materialized.addInvalidator(_supplier); @@ -362,9 +355,19 @@ public SQLFragment getFromSql(@NotNull SQLFragment selectQuery, String tableAlia TempTableTracker.track(name, materialized); - SQLFragment selectInto = new SQLFragment("SELECT * INTO \"" + temp.getName() + "\".\"" + name + "\"\nFROM (\n"); - selectInto.append(selectQuery); - selectInto.append("\n) _sql_"); + SQLFragment selectInto; + if (isSelectInto) + { + String sql = selectQuery.getSQL().replace("${NAME}", name); + List params = selectQuery.getParams(); + selectInto = new SQLFragment(sql,params); + } + else + { + selectInto = new SQLFragment("SELECT * INTO \"" + temp.getName() + "\".\"" + name + "\"\nFROM (\n"); + selectInto.append(selectQuery); + selectInto.append("\n) _sql_"); + } new SqlExecutor(_scope).execute(selectInto); try (var ignored = SpringActionController.ignoreSqlUpdates()) @@ -394,6 +397,53 @@ public SQLFragment getFromSql(@NotNull SQLFragment selectQuery, String tableAlia return sqlf; } + @Nullable + private Materialized getMaterialized(String txCacheKey) + { + Materialized materialized = null; + + synchronized (this) + { + if (_closed) + throw new IllegalStateException(); + + _countGetFromSql.incrementAndGet(); + + if (_scope.isTransactionActive()) + materialized = _map.get(txCacheKey); + + if (null == materialized) + materialized = _map.get(makeKey(null, null)); + } + + if (null != materialized) + { + boolean replace = false; + for (Invalidator i : materialized._invalidators) + { + CacheCheck cc = i.checkValid(materialized._created); + if (cc != CacheCheck.OK) + replace = true; + } + if (replace) + { + synchronized (this) + { + _map.remove(materialized._cacheKey); + materialized = null; + } + } + } + return materialized; + } + + + /* Do incremental update to existing cached data. There is no provision for deleting rows. */ + public void upsert() + { + + } + /** * To be consistent with CacheManager maxTimeToCache==0 means UNLIMITED, so we use maxTimeToCache==-1 to mean no caching, just materialize and return @@ -427,6 +477,7 @@ public static class Builder implements org.labkey.api.data.Builder _supplier = null; @@ -439,6 +490,13 @@ public Builder(String prefix, DbScope scope, SQLFragment select) _select = select; } + /** This property indicates that the SQLFragment is formatted as a SELECT INTO query (rather than a simple SELECT) */ + public Builder setIsSelectInto(boolean b) + { + _isSelectInto = b; + return this; + } + public Builder upToDateSql(SQLFragment uptodate) { _uptodate = uptodate; @@ -466,7 +524,7 @@ public Builder addIndex(String index) @Override public MaterializedQueryHelper build() { - return new MaterializedQueryHelper(_prefix, _scope, _select, _uptodate, _supplier, _indexes, _max, false); + return new MaterializedQueryHelper(_prefix, _scope, _select, _uptodate, _supplier, _indexes, _max, _isSelectInto); } } diff --git a/api/src/org/labkey/api/data/TableSelector.java b/api/src/org/labkey/api/data/TableSelector.java index e5cded12276..4b719f95b49 100644 --- a/api/src/org/labkey/api/data/TableSelector.java +++ b/api/src/org/labkey/api/data/TableSelector.java @@ -42,6 +42,7 @@ import java.util.List; import java.util.Map; import java.util.Set; +import java.util.stream.Collectors; import java.util.stream.Stream; public class TableSelector extends SqlExecutingSelector implements ResultsFactory @@ -129,7 +130,9 @@ private static Collection columnInfosList(@NotNull TableInfo table, if (select == ALL_COLUMNS) { - selectColumns = table.getColumns(); + selectColumns = table.getColumns().stream() + .filter(columnInfo -> !columnInfo.isUnselectable()) + .collect(Collectors.toList()); } else { diff --git a/experiment/src/org/labkey/experiment/ExperimentModule.java b/experiment/src/org/labkey/experiment/ExperimentModule.java index cde0bc340b4..80de413921b 100644 --- a/experiment/src/org/labkey/experiment/ExperimentModule.java +++ b/experiment/src/org/labkey/experiment/ExperimentModule.java @@ -153,6 +153,9 @@ public class ExperimentModule extends SpringModule implements SearchService.Docu public static final String EXPERIMENT_RUN_WEB_PART_NAME = "Experiment Runs"; + public static final String EXPERIMENTAL_LINEAGE_PARENT_LOOKUP = "lineage-parent-lookup"; + + @Override public String getName() { @@ -213,6 +216,11 @@ protected void init() AdminConsole.addExperimentalFeatureFlag(AppProps.EXPERIMENTAL_RESOLVE_PROPERTY_URI_COLUMNS, "Resolve property URIs as columns on experiment tables", "If a column is not found on an experiment table, attempt to resolve the column name as a Property URI and add it as a property column", false); + AdminConsole.addExperimentalFeatureFlag(EXPERIMENTAL_LINEAGE_PARENT_LOOKUP, + "Expose auto-generated lineage lookup columns in SampleType tables", + "Optimizes 'join' to parent samples/dataclass objects, when relationship is unique (one related row in parent table).", + false); + RoleManager.registerPermission(new DesignVocabularyPermission(), true); AttachmentService.get().registerAttachmentType(ExpRunAttachmentType.get()); diff --git a/experiment/src/org/labkey/experiment/api/ClosureQueryHelper.java b/experiment/src/org/labkey/experiment/api/ClosureQueryHelper.java new file mode 100644 index 00000000000..acf94f9cd3b --- /dev/null +++ b/experiment/src/org/labkey/experiment/api/ClosureQueryHelper.java @@ -0,0 +1,534 @@ +package org.labkey.experiment.api; + +import org.apache.commons.lang3.StringUtils; +import org.jetbrains.annotations.NotNull; +import org.jetbrains.annotations.Nullable; +import org.labkey.api.data.AbstractForeignKey; +import org.labkey.api.data.BaseColumnInfo; +import org.labkey.api.data.ColumnInfo; +import org.labkey.api.data.Container; +import org.labkey.api.data.CoreSchema; +import org.labkey.api.data.DbSchema; +import org.labkey.api.data.DbScope; +import org.labkey.api.data.JdbcType; +import org.labkey.api.data.MaterializedQueryHelper; +import org.labkey.api.data.MutableColumnInfo; +import org.labkey.api.data.SQLFragment; +import org.labkey.api.data.SqlExecutor; +import org.labkey.api.data.TableInfo; +import org.labkey.api.data.TempTableTracker; +import org.labkey.api.data.VirtualTable; +import org.labkey.api.data.dialect.SqlDialect; +import org.labkey.api.exp.api.ExpDataClass; +import org.labkey.api.exp.api.ExpObject; +import org.labkey.api.exp.api.ExpSampleType; +import org.labkey.api.query.ExprColumn; +import org.labkey.api.query.FieldKey; +import org.labkey.api.query.FilteredTable; +import org.labkey.api.query.QueryForeignKey; +import org.labkey.api.query.SchemaKey; +import org.labkey.api.query.UserSchema; +import org.labkey.api.security.User; +import org.labkey.api.util.HeartBeat; +import org.labkey.api.util.StringExpression; + +import java.util.Collection; +import java.util.Collections; +import java.util.HashMap; +import java.util.LinkedHashMap; +import java.util.Map; +import java.util.Objects; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicInteger; + +import static org.labkey.api.exp.api.ExpProtocol.ApplicationType.ExperimentRunOutput; + + +public class ClosureQueryHelper +{ + final static long CACHE_INVALIDATION_INTERVAL = TimeUnit.MINUTES.toMillis(5); + final static long CACHE_LRU_AGE_OUT_INTERVAL = TimeUnit.MINUTES.toMillis(30); + + + + /* TODO/CONSIDER every SampleType and Dataclass should have a unique ObjectId so it can be stored as an in lineage tables (e.g. edge/closure tables) */ + + record ClosureTable(MaterializedQueryHelper helper, AtomicInteger counter, TableType type, String lsid) {}; + + static final Map queryHelpers = Collections.synchronizedMap(new HashMap<>()); + // use this as a separate LRU implementation, because I only want to track calls to getValueSql() not other calls to queryHelpers.get() + static final Map lruQueryHelpers = new LinkedHashMap(100,0.75f,true); + + + static final int MAX_LINEAGE_LOOKUP_DEPTH = 10; + + static String pgMaterialClosureCTE = String.format(""" + WITH RECURSIVE CTE_ AS ( + + SELECT + RowId AS Start_, + ObjectId as End_, + '/' || CAST(ObjectId AS VARCHAR) || '/' as Path_, + 0 as Depth_ + /*FROM*/ + + UNION ALL + + SELECT CTE_.Start_, Edge.FromObjectId as End_, CTE_.Path_ || CAST(Edge.FromObjectId AS VARCHAR) || '/' as Path_, Depth_ + 1 as Depth_ + FROM CTE_ INNER JOIN exp.Edge ON CTE_.End_ = Edge.ToObjectId + WHERE Depth_ < %d AND 0 = POSITION('/' || CAST(Edge.FromObjectId AS VARCHAR) || '/' IN Path_) + + ) + """, MAX_LINEAGE_LOOKUP_DEPTH); + + static String pgMaterialClosureSql = """ + SELECT Start_, CASE WHEN COUNT(*) = 1 THEN MIN(rowId) ELSE -1 * COUNT(*) END AS rowId, targetId + /*INTO*/ + FROM ( + SELECT Start_, End_, + COALESCE(material.rowid, data.rowid) as rowId, + COALESCE('m' || CAST(materialsource.rowid AS VARCHAR), 'd' || CAST(dataclass.rowid AS VARCHAR)) as targetId + FROM CTE_ + LEFT OUTER JOIN exp.material ON End_ = material.objectId LEFT OUTER JOIN exp.materialsource ON material.cpasType = materialsource.lsid + LEFT OUTER JOIN exp.data on End_ = data.objectId LEFT OUTER JOIN exp.dataclass ON data.cpasType = dataclass.lsid + WHERE Depth_ > 0 AND materialsource.rowid IS NOT NULL OR dataclass.rowid IS NOT NULL) _inner_ + GROUP BY targetId, Start_ + """; + + + static String mssqlMaterialClosureCTE = String.format(""" + WITH CTE_ AS ( + + SELECT + RowId AS Start_, + ObjectId as End_, + '/' + CAST(ObjectId AS VARCHAR(MAX)) + '/' as Path_, + 0 as Depth_ + /*FROM*/ + + UNION ALL + + SELECT CTE_.Start_, Edge.FromObjectId as End_, CTE_.Path_ + CAST(Edge.FromObjectId AS VARCHAR) + '/' as Path_, Depth_ + 1 as Depth_ + FROM CTE_ INNER JOIN exp.Edge ON CTE_.End_ = Edge.ToObjectId + WHERE Depth_ < %d AND 0 = CHARINDEX('/' + CAST(Edge.FromObjectId AS VARCHAR) + '/', Path_) + ) + """, (MAX_LINEAGE_LOOKUP_DEPTH)); + + static String mssqlMaterialClosureSql = """ + SELECT Start_, CASE WHEN COUNT(*) = 1 THEN MIN(rowId) ELSE -1 * COUNT(*) END AS rowId, targetId + /*INTO*/ + FROM ( + SELECT Start_, End_, + COALESCE(material.rowid, data.rowid) as rowId, + COALESCE('m' + CAST(materialsource.rowid AS VARCHAR), 'd' + CAST(dataclass.rowid AS VARCHAR)) as targetId + FROM CTE_ + LEFT OUTER JOIN exp.material ON End_ = material.objectId LEFT OUTER JOIN exp.materialsource ON material.cpasType = materialsource.lsid + LEFT OUTER JOIN exp.data on End_ = data.objectId LEFT OUTER JOIN exp.dataclass ON data.cpasType = dataclass.lsid + WHERE Depth_ > 0 AND materialsource.rowid IS NOT NULL OR dataclass.rowid IS NOT NULL) _inner_ + GROUP BY targetId, Start_ + """; + + + static SQLFragment selectIntoSql(SqlDialect d, SQLFragment from, @Nullable String tempTable) + { + String cte = d.isPostgreSQL() ? pgMaterialClosureCTE : mssqlMaterialClosureCTE; + String select = d.isPostgreSQL() ? pgMaterialClosureSql : mssqlMaterialClosureSql; + + String[] cteParts = StringUtils.splitByWholeSeparator(cte,"/*FROM*/"); + assert cteParts.length == 2; + + String into = " INTO temp.${NAME} "; + if (null != tempTable) + into = " INTO temp." + tempTable + " "; + String[] selectIntoParts = StringUtils.splitByWholeSeparator(select,"/*INTO*/"); + assert selectIntoParts.length == 2; + + return new SQLFragment() + .append(cteParts[0]).append(" ").append(from).append(" ").append(cteParts[1]) + .append(selectIntoParts[0]).append(into).append(selectIntoParts[1]); + } + + + static SQLFragment selectSql(SqlDialect d, SQLFragment from) + { + String cte = d.isPostgreSQL() ? pgMaterialClosureCTE : mssqlMaterialClosureCTE; + String select = d.isPostgreSQL() ? pgMaterialClosureSql : mssqlMaterialClosureSql; + + String[] cteParts = StringUtils.splitByWholeSeparator(cte,"/*FROM*/"); + assert cteParts.length == 2; + + return new SQLFragment(cteParts[0]).append(from).append(cteParts[1]).append(select); + } + + + /* + * This can be used to add a column directly to a exp table, or to create a column + * in an intermediate fake lookup table + */ + static MutableColumnInfo createLineageLookupColumn(final ColumnInfo fkRowId, ExpObject source, ExpObject target) + { + if (!(source instanceof ExpSampleType) && !(source instanceof ExpDataClass)) + throw new IllegalStateException(); + if (!(target instanceof ExpSampleType) && !(target instanceof ExpDataClass)) + throw new IllegalStateException(); + + final TableType sourceType = source instanceof ExpSampleType ? TableType.SampleType : TableType.DataClass; + final TableType targetType = target instanceof ExpSampleType ? TableType.SampleType : TableType.DataClass; + + TableInfo parentTable = fkRowId.getParentTable(); + var ret = new ExprColumn(parentTable, target.getName(), new SQLFragment("#ERROR#"), JdbcType.INTEGER) + { + @Override + public SQLFragment getValueSql(String tableAlias) + { + SQLFragment objectId = fkRowId.getValueSql(tableAlias); + String sourceLsid = source.getLSID(); + if (sourceLsid == null) + return new SQLFragment(" NULL "); + return ClosureQueryHelper.getValueSql(sourceType, sourceLsid, objectId, target); + } + }; + ret.setLabel(target.getName()); + UserSchema schema = Objects.requireNonNull(parentTable.getUserSchema()); + var qfk = new QueryForeignKey.Builder(schema, parentTable.getContainerFilter()).table(target.getName()).key("rowid"); + if (sourceType != targetType) + qfk.schema(targetType.schemaKey); + ret.setFk(qfk); + return ret; + } + + + public static SQLFragment getValueSql(TableType type, String sourceLSID, SQLFragment objectId, ExpObject target) + { + if (target instanceof ExpSampleType st) + return getValueSql(type, sourceLSID, objectId, "m" + st.getRowId()); + if (target instanceof ExpDataClass dc) + return getValueSql(type, sourceLSID, objectId, "d" + dc.getRowId()); + throw new IllegalStateException(); + } + + + private static SQLFragment getValueSql(TableType type, String sourceLSID, SQLFragment objectId, String targetId) + { + MaterializedQueryHelper helper = getClosureHelper(type, sourceLSID, true); + + return new SQLFragment() + .append("(SELECT rowId FROM ") + .append(helper.getFromSql("CLOS", null)) + .append(" WHERE targetId='").append(targetId).append("'") + .append(" AND Start_=").append(objectId) + .append(")"); + } + + + static final AtomicInteger temptableNumber = new AtomicInteger(); + + private static void incrementalRecompute(String sourceLSID, SQLFragment from) + { + // if there's nothing cached, we don't need to do incremental + MaterializedQueryHelper helper = getClosureHelper(null, sourceLSID, false); + if (null == helper || !helper.isCached(null)) + return; + + TempTableTracker ttt = null; + try + { + Object ref = new Object(); + String tempTableName = "closinc_"+temptableNumber.incrementAndGet(); + ttt = TempTableTracker.track(tempTableName, ref); + SQLFragment selectInto = selectIntoSql(getScope().getSqlDialect(), from, tempTableName); + new SqlExecutor(getScope()).execute(selectInto); + + SQLFragment upsert; + if (getScope().getSqlDialect().isPostgreSQL()) + { + upsert = new SQLFragment() + .append("INSERT INTO temp.${NAME} (Start_, rowId, targetid)\n") + .append("SELECT Start_, RowId, targetId FROM temp.").append(tempTableName).append(" TMP\n") + .append("ON CONFLICT(Start_,targetId) DO UPDATE SET rowId = EXCLUDED.rowId;"); + } + else + { + upsert = new SQLFragment() + .append("MERGE temp.${NAME} AS Target\n") + .append("USING (SELECT Start_, RowId, targetId FROM temp.").append(tempTableName).append(") AS Source ON Target.Start_=Source.Start_ AND Target.targetid=Source.targetId\n") + .append("WHEN MATCHED THEN UPDATE SET Target.targetId = Source.targetId\n") + .append("WHEN NOT MATCHED THEN INSERT (Start_, rowId, targetid) VALUES (Source.Start_, Source.rowId, Source.targetId);"); + } + + helper.upsert(upsert); + } + catch (Exception x) + { + invalidate(sourceLSID); + throw x; + } + finally + { + if (null != ttt) + ttt.delete(); + } + } + + + public static void invalidateAll() + { + synchronized (queryHelpers) + { + for (var c : queryHelpers.values()) + { + c.counter.incrementAndGet(); + c.helper.uncache(null); + } + } + } + + + public static void invalidateMaterialsForRun(String sourceTypeLsid, int runId) + { + var tx = getScope().getCurrentTransaction(); + if (null != tx) + { + tx.addCommitTask(() -> invalidateMaterialsForRun(sourceTypeLsid, runId), DbScope.CommitTaskOption.POSTCOMMIT); + return; + } + + SQLFragment seedFrom = new SQLFragment() + .append("FROM (SELECT m.RowId, m.ObjectId FROM exp.material m\n") + .append("INNER JOIN exp.MaterialInput mi ON m.rowId = mi.materialId\n") + .append("INNER JOIN exp.ProtocolApplication pa ON mi.TargetApplicationId = pa.RowId\n") + .append("WHERE pa.RunId = ").append(runId) + .append(" AND m.cpasType = ? ").add(sourceTypeLsid) + .append(" AND pa.CpasType = '").append(ExperimentRunOutput.name()).append("') _seed_ "); + incrementalRecompute(sourceTypeLsid, seedFrom); + } + + + private static MaterializedQueryHelper getClosureHelper(TableType type, String sourceLSID, boolean computeIfAbsent) + { + ClosureTable closure; + + if (!computeIfAbsent) + { + closure = queryHelpers.get(sourceLSID); + return null==closure ? null : closure.helper; + } + + if (null == type) + throw new IllegalStateException(); + + closure = queryHelpers.computeIfAbsent(sourceLSID, cpasType -> + { + SQLFragment from = new SQLFragment(" FROM exp.Material WHERE Material.cpasType = ? ").add(cpasType); + SQLFragment selectInto = selectIntoSql(getScope().getSqlDialect(), from, null); + + var helper = new MaterializedQueryHelper.Builder("closure", DbSchema.getTemp().getScope(), selectInto) + .setIsSelectInto(true) + .addIndex("CREATE UNIQUE INDEX uq_${NAME} ON temp.${NAME} (targetId,Start_)") + .maxTimeToCache(CACHE_INVALIDATION_INTERVAL) + .addInvalidCheck(() -> getInvalidationCounterString(sourceLSID)) + .build(); + return new ClosureTable(helper, new AtomicInteger(), type, sourceLSID); + }); + + // update LRU + synchronized (lruQueryHelpers) + { + lruQueryHelpers.put(sourceLSID, HeartBeat.currentTimeMillis()); + checkStaleEntries(); + } + + return closure.helper; + } + + + private static void checkStaleEntries() + { + synchronized (lruQueryHelpers) + { + if (lruQueryHelpers.isEmpty()) + return; + var oldestEntry = lruQueryHelpers.entrySet().iterator().next(); + if (HeartBeat.currentTimeMillis() - oldestEntry.getValue() < CACHE_LRU_AGE_OUT_INTERVAL) + return; + queryHelpers.remove(oldestEntry.getKey()); + lruQueryHelpers.remove(oldestEntry.getKey()); + } + } + + + private static void invalidate(String sourceLSID) + { + var closure = queryHelpers.get(sourceLSID); + if (null != closure) + closure.counter.incrementAndGet(); + } + + + private static String getInvalidationCounterString(String sourceLSID) + { + var closure = queryHelpers.get(sourceLSID); + return null==closure ? "-1" : String.valueOf(closure.counter.get()); + } + + + private static DbScope getScope() + { + return CoreSchema.getInstance().getScope(); + } + + + + + + + + /* + * Code to create the lineage parent lookup column and intermediate lookups that use ClosureQueryHelper + */ + + public static MutableColumnInfo createLineageLookupColumnInfo(String columnName, FilteredTable parent, ColumnInfo rowid, ExpObject source) + { + MutableColumnInfo wrappedRowId = parent.wrapColumn(columnName, rowid); + wrappedRowId.setIsUnselectable(true); + wrappedRowId.setReadOnly(true); + wrappedRowId.setCalculated(true); + wrappedRowId.setRequired(false); + wrappedRowId.setFk(new AbstractForeignKey(parent.getUserSchema(),parent.getContainerFilter()) + { + @Override + public TableInfo getLookupTableInfo() + { + return new LineageLookupTypesTableInfo(parent.getUserSchema(), source); + } + + @Override + public @Nullable ColumnInfo createLookupColumn(ColumnInfo parent, String displayField) + { + ColumnInfo lk = getLookupTableInfo().getColumn(displayField); + if (null == lk) + return null; + var ret = new ExprColumn(parent.getParentTable(), new FieldKey(parent.getFieldKey(),lk.getName()), null, JdbcType.INTEGER) + { + @Override + public SQLFragment getValueSql(String tableAlias) + { + return parent.getValueSql(tableAlias); + } + }; + ret.setFk(lk.getFk()); + return ret; + } + + @Override + public StringExpression getURL(ColumnInfo parent) + { + return null; + } + }); + return wrappedRowId; + } + + + enum TableType + { + SampleType("Materials", SchemaKey.fromParts("exp","materials") ) + { + @Override + Collection getInstances(Container c, User u) + { + return SampleTypeServiceImpl.get().getSampleTypes(c, u,false); + } + @Override + ExpObject getInstance(Container c, User u, String name) + { + return SampleTypeServiceImpl.get().getSampleType(c, u, name); + } + }, + DataClass("Data", SchemaKey.fromParts("exp","data") ) + { + @Override + Collection getInstances(Container c, User u) + { + return ExperimentServiceImpl.get().getDataClasses(c, u,false); + } + @Override + ExpObject getInstance(Container c, User u, String name) + { + return ExperimentServiceImpl.get().getDataClass(c, u, name); + } + }; + + final String lookupName; + final SchemaKey schemaKey; + + TableType(String lookupName, SchemaKey schemaKey) + { + this.lookupName = lookupName; + this.schemaKey = schemaKey; + } + + abstract Collection getInstances(Container c, User u); + abstract ExpObject getInstance(Container c, User u, String name); + }; + + + private static class LineageLookupTypesTableInfo extends VirtualTable + { + LineageLookupTypesTableInfo(UserSchema userSchema, ExpObject source) + { + super(userSchema.getDbSchema(), "LineageLookupTypes",userSchema); + + for (var lk : TableType.values()) + { + var col = new BaseColumnInfo(lk.lookupName, this, JdbcType.INTEGER); + col.setIsUnselectable(true); + col.setFk(new AbstractForeignKey(getUserSchema(),null) + { + @Override + public @Nullable ColumnInfo createLookupColumn(ColumnInfo parent, String displayField) + { + if (null == displayField) + return null; + var target = lk.getInstance(_userSchema.getContainer(), _userSchema.getUser(), displayField); + if (null == target) + return null; + return ClosureQueryHelper.createLineageLookupColumn(parent, source, target); + } + + @Override + public TableInfo getLookupTableInfo() + { + return new LineageLookupTableInfo(userSchema, source, lk); + } + + @Override + public StringExpression getURL(ColumnInfo parent) + { + return null; + } + }); + addColumn(col); + } + } + } + + + private static class LineageLookupTableInfo extends VirtualTable + { + LineageLookupTableInfo(UserSchema userSchema, ExpObject source, TableType type) + { + super(userSchema.getDbSchema(), "Lineage Lookup", userSchema); + ColumnInfo wrap = new BaseColumnInfo("rowid", this, JdbcType.INTEGER); + for (var target : type.getInstances(_userSchema.getContainer(), _userSchema.getUser())) + addColumn(ClosureQueryHelper.createLineageLookupColumn(wrap, source, target)); + } + + @Override + public @NotNull SQLFragment getFromSQL() + { + throw new IllegalStateException(); + } + } + +} diff --git a/experiment/src/org/labkey/experiment/api/ExpMaterialTableImpl.java b/experiment/src/org/labkey/experiment/api/ExpMaterialTableImpl.java index 4da5d3cacdc..9001ff89fde 100644 --- a/experiment/src/org/labkey/experiment/api/ExpMaterialTableImpl.java +++ b/experiment/src/org/labkey/experiment/api/ExpMaterialTableImpl.java @@ -69,6 +69,7 @@ import org.labkey.api.security.permissions.Permission; import org.labkey.api.security.permissions.ReadPermission; import org.labkey.api.security.permissions.UpdatePermission; +import org.labkey.api.settings.AppProps; import org.labkey.api.util.PageFlowUtil; import org.labkey.api.util.Pair; import org.labkey.api.util.StringExpression; @@ -76,6 +77,7 @@ import org.labkey.api.view.ViewContext; import org.labkey.experiment.ExpDataIterators; import org.labkey.experiment.ExpDataIterators.AliasDataIteratorBuilder; +import org.labkey.experiment.ExperimentModule; import org.labkey.experiment.controllers.exp.ExperimentController; import java.io.IOException; @@ -670,6 +672,12 @@ protected ContainerFilter getLookupContainerFilter() setTitleColumn(Column.Name.toString()); setDefaultVisibleColumns(defaultCols); + + if (null != _ss && AppProps.getInstance().isExperimentalFeatureEnabled(ExperimentModule.EXPERIMENTAL_LINEAGE_PARENT_LOOKUP)) + { + MutableColumnInfo lineageLookup = ClosureQueryHelper.createLineageLookupColumnInfo("Ancestor Lookups Placeholder", this, _rootTable.getColumn("rowid"), _ss); + addColumn(lineageLookup); + } } @Override diff --git a/experiment/src/org/labkey/experiment/api/ExperimentServiceImpl.java b/experiment/src/org/labkey/experiment/api/ExperimentServiceImpl.java index 29aa441928d..f144241ffab 100644 --- a/experiment/src/org/labkey/experiment/api/ExperimentServiceImpl.java +++ b/experiment/src/org/labkey/experiment/api/ExperimentServiceImpl.java @@ -2900,7 +2900,7 @@ public SyncRunEdgesTask(int runId, Integer runObjectId, String runLsid, Containe public void run() { if (_runObjectId !=null && _runLsid != null && _runContainer != null) - syncRunEdges(_runId, _runObjectId, _runLsid, _runContainer); + new SyncRunEdges(_runId, _runObjectId, _runLsid, _runContainer).sync(null); else syncRunEdges(_runId); } @@ -2958,211 +2958,274 @@ public void syncRunEdges(int runId) { ExpRun run = getExpRun(runId); if (run != null) - syncRunEdges(run); + new SyncRunEdges(run).sync(null); } + @Override public void syncRunEdges(ExpRun run) { - syncRunEdges(run.getRowId(), run.getObjectId(), run.getLSID(), run.getContainer()); + new SyncRunEdges(run).sync(null); } + @Override public void syncRunEdges(Collection runs) { + Map cpasTypeToObjectId = new HashMap<>(); + for (ExpRun run : runs) - syncRunEdges(run.getRowId(), run.getObjectId(), run.getLSID(), run.getContainer()); + { + new SyncRunEdges(run).sync(cpasTypeToObjectId); + } } - public void syncRunEdges(int runId, Integer runObjectId, String runLsid, Container runContainer) - { - syncRunEdges(runId, runObjectId, runLsid, runContainer, true, false, null); - } - private void syncRunEdges(int runId, Integer runObjectId, String runLsid, Container runContainer, boolean deleteFirst, boolean verifyEdgesNoInsert, @Nullable Map cpasTypeToObjectId) + /* syncRunEdges() has too many boolean parameters, so here's a mini builder */ + + class SyncRunEdges { - // don't do any updates if we are just verifying - if (verifyEdgesNoInsert) - deleteFirst = false; + final int runId; + Integer runObjectId; + final String runLsid; + final Container runContainer; + boolean deleteFirst = true; + boolean verifyEdgesNoInsert=false; + boolean invalidateClosureCache=true; - CPUTimer timer = new CPUTimer("sync edges"); - timer.start(); + SyncRunEdges(ExpRun run) + { + this.runId = run.getRowId(); + this.runObjectId = run.getObjectId(); + this.runLsid = run.getLSID(); + this.runContainer = run.getContainer(); + } - LOG.debug((verifyEdgesNoInsert ? "Verifying" : "Rebuilding") + " edges for runId " + runId); - try (DbScope.Transaction tx = getExpSchema().getScope().ensureTransaction()) + SyncRunEdges(int runId, Integer runObjectId, String runLsid, Container runContainer) { - // NOTE: Originally, we just filtered exp.data by runId. This works for most runs but includes intermediate exp.data nodes and caused the ExpTest to fail - SQLFragment datas = new SQLFragment() - .append("SELECT d.Container, d.LSID, d.ObjectId, pa.CpasType AS pa_cpas_type FROM exp.Data d\n") - .append("INNER JOIN exp.DataInput di ON d.rowId = di.dataId\n") - .append("INNER JOIN exp.ProtocolApplication pa ON di.TargetApplicationId = pa.RowId\n") - .append("WHERE pa.RunId = ").append(runId).append(" AND pa.CpasType IN ('").append(ExperimentRun.name()).append("','").append(ExperimentRunOutput.name()).append("')"); + this.runId = runId; + this.runObjectId = runObjectId; + this.runLsid = runLsid; + this.runContainer = runContainer; + } - Collection> fromDataLsids = new ArrayList<>(); - Collection> toDataLsids = new ArrayList<>(); - new SqlSelector(getSchema(), datas).forEachMap(row -> { - if (ExperimentRun.name().equals(row.get("pa_cpas_type"))) - fromDataLsids.add(row); - else - toDataLsids.add(row); - }); - if (LOG.isDebugEnabled()) - { - if (!fromDataLsids.isEmpty()) - LOG.debug(" fromDataLsids:\n " + StringUtils.join(fromDataLsids, "\n ")); - if (!toDataLsids.isEmpty()) - LOG.debug(" toDataLsids:\n " + StringUtils.join(toDataLsids, "\n ")); - } + SyncRunEdges deleteFirst(boolean d) + { + deleteFirst = d; + return this; + } - SQLFragment materials = new SQLFragment() - .append("SELECT m.Container, m.LSID, m.CpasType, m.ObjectId, pa.CpasType AS pa_cpas_type FROM exp.material m\n") - .append("INNER JOIN exp.MaterialInput mi ON m.rowId = mi.materialId\n") - .append("INNER JOIN exp.ProtocolApplication pa ON mi.TargetApplicationId = pa.RowId\n") - .append("WHERE pa.RunId = ").append(runId).append(" AND pa.CpasType IN ('").append(ExperimentRun.name()).append("','").append(ExperimentRunOutput.name()).append("')"); + SyncRunEdges verifyEdgesNoInsert(boolean v) + { + verifyEdgesNoInsert = v; + return this; + } - Collection> fromMaterialLsids = new ArrayList<>(); - Collection> toMaterialLsids = new ArrayList<>(); - new SqlSelector(getSchema(), materials).forEachMap(row -> { - if (ExperimentRun.name().equals(row.get("pa_cpas_type"))) - fromMaterialLsids.add(row); - else - toMaterialLsids.add(row); - }); + SyncRunEdges invalidateClosureCache(boolean i) + { + invalidateClosureCache = i; + return this; + } - Set> provenanceStartingInputs = emptySet(); - Set> provenanceFinalOutputs = emptySet(); + void sync(@Nullable Map cpasTypeToObjectId) + { + // don't do any updates if we are just verifying + if (verifyEdgesNoInsert) + deleteFirst = false; - ProvenanceService pvs = ProvenanceService.get(); - ProtocolApplication startProtocolApp = getStartingProtocolApplication(runId); - if (null != startProtocolApp) - { - provenanceStartingInputs = pvs.getProvenanceObjectIds(startProtocolApp.getRowId()); - } + CPUTimer timer = new CPUTimer("sync edges"); + timer.start(); - ProtocolApplication finalProtocolApp = getFinalProtocolApplication(runId); - if (null != finalProtocolApp) + LOG.debug((verifyEdgesNoInsert ? "Verifying" : "Rebuilding") + " edges for runId " + runId); + try (DbScope.Transaction tx = getExpSchema().getScope().ensureTransaction()) { - provenanceFinalOutputs = pvs.getProvenanceObjectIds(finalProtocolApp.getRowId()); - } + // NOTE: Originally, we just filtered exp.data by runId. This works for most runs but includes intermediate exp.data nodes and caused the ExpTest to fail + SQLFragment datas = new SQLFragment() + .append("SELECT d.Container, d.LSID, d.ObjectId, pa.CpasType AS pa_cpas_type FROM exp.Data d\n") + .append("INNER JOIN exp.DataInput di ON d.rowId = di.dataId\n") + .append("INNER JOIN exp.ProtocolApplication pa ON di.TargetApplicationId = pa.RowId\n") + .append("WHERE pa.RunId = ").append(runId).append(" AND pa.CpasType IN ('").append(ExperimentRun.name()).append("','").append(ExperimentRunOutput.name()).append("')"); - // delete all existing edges for this run - if (deleteFirst) - removeEdgesForRun(runId); - - int edgeCount = fromDataLsids.size() + fromMaterialLsids.size() + toDataLsids.size() + toMaterialLsids.size() + provenanceStartingInputs.size() + provenanceFinalOutputs.size(); - LOG.debug(String.format(" edge counts: input data=%d, input materials=%d, output data=%d, output materials=%d, input prov=%d, output prov=%d, total=%d", - fromDataLsids.size(), fromMaterialLsids.size(), toDataLsids.size(), toMaterialLsids.size(), provenanceStartingInputs.size(), provenanceFinalOutputs.size(), edgeCount)); - - if (edgeCount > 0) - { - // ensure the run has an exp.object - if (null == runObjectId || 0 == runObjectId) + Collection> fromDataLsids = new ArrayList<>(); + Collection> toDataLsids = new ArrayList<>(); + new SqlSelector(getSchema(), datas).forEachMap(row -> { + if (ExperimentRun.name().equals(row.get("pa_cpas_type"))) + fromDataLsids.add(row); + else + toDataLsids.add(row); + }); + if (LOG.isDebugEnabled()) { - if (LOG.isDebugEnabled()) - { - OntologyObject runObj = OntologyManager.getOntologyObject(runContainer, runLsid); - if (runObj == null) - LOG.debug(" run exp.object is null, creating: " + runLsid); - } - if (!verifyEdgesNoInsert) - runObjectId = OntologyManager.ensureObject(runContainer, runLsid, (Integer) null); + if (!fromDataLsids.isEmpty()) + LOG.debug(" fromDataLsids:\n " + StringUtils.join(fromDataLsids, "\n ")); + if (!toDataLsids.isEmpty()) + LOG.debug(" toDataLsids:\n " + StringUtils.join(toDataLsids, "\n ")); } - Map> allDatasByLsid = new HashMap<>(); - fromDataLsids.forEach(row -> allDatasByLsid.put((String) row.get("lsid"), row)); - toDataLsids.forEach(row -> allDatasByLsid.put((String) row.get("lsid"), row)); - if (!verifyEdgesNoInsert) - ensureNodeObjects(getTinfoData(), allDatasByLsid, cpasTypeToObjectId != null ? cpasTypeToObjectId : new HashMap<>()); - - Map> allMaterialsByLsid = new HashMap<>(); - fromMaterialLsids.forEach(row -> allMaterialsByLsid.put((String) row.get("lsid"), row)); - toMaterialLsids.forEach(row -> allMaterialsByLsid.put((String) row.get("lsid"), row)); - if (!verifyEdgesNoInsert) - ensureNodeObjects(getTinfoMaterial(), allMaterialsByLsid, cpasTypeToObjectId != null ? cpasTypeToObjectId : new HashMap<>()); - - List> params = new ArrayList<>(edgeCount); + SQLFragment materials = new SQLFragment() + .append("SELECT m.Container, m.LSID, m.CpasType, m.ObjectId, pa.CpasType AS pa_cpas_type FROM exp.material m\n") + .append("INNER JOIN exp.MaterialInput mi ON m.rowId = mi.materialId\n") + .append("INNER JOIN exp.ProtocolApplication pa ON mi.TargetApplicationId = pa.RowId\n") + .append("WHERE pa.RunId = ").append(runId).append(" AND pa.CpasType IN ('").append(ExperimentRun.name()).append("','").append(ExperimentRunOutput.name()).append("')"); + + Set toCpasTypes = new HashSet<>(); + Collection> fromMaterialLsids = new ArrayList<>(); + Collection> toMaterialLsids = new ArrayList<>(); + new SqlSelector(getSchema(), materials).forEachMap(row -> { + if (ExperimentRun.name().equals(row.get("pa_cpas_type"))) + { + fromMaterialLsids.add(row); + } + else + { + toMaterialLsids.add(row); + toCpasTypes.add((String)row.get("cpastype")); + } + }); - // - // from lsid -> run lsid - // + Set> provenanceStartingInputs = emptySet(); + Set> provenanceFinalOutputs = emptySet(); - Set seen = new HashSet<>(); - for (Map fromDataLsid : fromDataLsids) + ProvenanceService pvs = ProvenanceService.get(); + ProtocolApplication startProtocolApp = getStartingProtocolApplication(runId); + if (null != startProtocolApp) { - assert null != fromDataLsid.get("objectid"); - int objectid = (Integer)fromDataLsid.get("objectid"); - if (seen.add(objectid)) - prepEdgeForInsert(params, objectid, runObjectId, runId); + provenanceStartingInputs = pvs.getProvenanceObjectIds(startProtocolApp.getRowId()); } - for (Map fromMaterialLsid : fromMaterialLsids) + ProtocolApplication finalProtocolApp = getFinalProtocolApplication(runId); + if (null != finalProtocolApp) { - assert null != fromMaterialLsid.get("objectid"); - int objectid = (Integer)fromMaterialLsid.get("objectid"); - if (seen.add(objectid)) - prepEdgeForInsert(params, objectid, runObjectId, runId); + provenanceFinalOutputs = pvs.getProvenanceObjectIds(finalProtocolApp.getRowId()); } - if (!provenanceStartingInputs.isEmpty()) + // delete all existing edges for this run + if (deleteFirst) + removeEdgesForRun(runId); + + int edgeCount = fromDataLsids.size() + fromMaterialLsids.size() + toDataLsids.size() + toMaterialLsids.size() + provenanceStartingInputs.size() + provenanceFinalOutputs.size(); + LOG.debug(String.format(" edge counts: input data=%d, input materials=%d, output data=%d, output materials=%d, input prov=%d, output prov=%d, total=%d", + fromDataLsids.size(), fromMaterialLsids.size(), toDataLsids.size(), toMaterialLsids.size(), provenanceStartingInputs.size(), provenanceFinalOutputs.size(), edgeCount)); + + if (edgeCount > 0) { - for (Pair pair : provenanceStartingInputs) + // ensure the run has an exp.object + if (null == runObjectId || 0 == runObjectId) { - Integer fromId = pair.first; - if (null != fromId) + if (LOG.isDebugEnabled()) { - if (seen.add(fromId)) - prepEdgeForInsert(params, fromId, runObjectId, runId); + OntologyObject runObj = OntologyManager.getOntologyObject(runContainer, runLsid); + if (runObj == null) + LOG.debug(" run exp.object is null, creating: " + runLsid); } + if (!verifyEdgesNoInsert) + runObjectId = OntologyManager.ensureObject(runContainer, runLsid, (Integer) null); } - } - // - // run lsid -> to lsid - // + Map> allDatasByLsid = new HashMap<>(); + fromDataLsids.forEach(row -> allDatasByLsid.put((String) row.get("lsid"), row)); + toDataLsids.forEach(row -> allDatasByLsid.put((String) row.get("lsid"), row)); + if (!verifyEdgesNoInsert) + ensureNodeObjects(getTinfoData(), allDatasByLsid, cpasTypeToObjectId != null ? cpasTypeToObjectId : new HashMap<>()); + + Map> allMaterialsByLsid = new HashMap<>(); + fromMaterialLsids.forEach(row -> allMaterialsByLsid.put((String) row.get("lsid"), row)); + toMaterialLsids.forEach(row -> allMaterialsByLsid.put((String) row.get("lsid"), row)); + if (!verifyEdgesNoInsert) + ensureNodeObjects(getTinfoMaterial(), allMaterialsByLsid, cpasTypeToObjectId != null ? cpasTypeToObjectId : new HashMap<>()); - seen = new HashSet<>(); - for (Map toDataLsid : toDataLsids) - { - int objectid = (Integer)toDataLsid.get("objectid"); - if (seen.add(objectid)) - prepEdgeForInsert(params, runObjectId, objectid, runId); - } + List> params = new ArrayList<>(edgeCount); - for (Map toMaterialLsid : toMaterialLsids) - { - int objectid = (Integer)toMaterialLsid.get("objectid"); - if (seen.add(objectid)) - prepEdgeForInsert(params, runObjectId, objectid, runId); - } + // + // from lsid -> run lsid + // - if (!provenanceFinalOutputs.isEmpty()) - { - for (Pair pair : provenanceFinalOutputs) + Set seen = new HashSet<>(); + for (Map fromDataLsid : fromDataLsids) + { + assert null != fromDataLsid.get("objectid"); + int objectid = (Integer)fromDataLsid.get("objectid"); + if (seen.add(objectid)) + prepEdgeForInsert(params, objectid, runObjectId, runId); + } + + for (Map fromMaterialLsid : fromMaterialLsids) + { + assert null != fromMaterialLsid.get("objectid"); + int objectid = (Integer)fromMaterialLsid.get("objectid"); + if (seen.add(objectid)) + prepEdgeForInsert(params, objectid, runObjectId, runId); + } + + if (!provenanceStartingInputs.isEmpty()) { - Integer toObjectId = pair.second; - if (null != toObjectId) + for (Pair pair : provenanceStartingInputs) { - if (seen.add(toObjectId)) - prepEdgeForInsert(params, runObjectId, toObjectId, runId); + Integer fromId = pair.first; + if (null != fromId) + { + if (seen.add(fromId)) + prepEdgeForInsert(params, fromId, runObjectId, runId); + } } } - } - if (verifyEdgesNoInsert) - verifyEdges(runId, runObjectId, params); + // + // run lsid -> to lsid + // + + seen = new HashSet<>(); + for (Map toDataLsid : toDataLsids) + { + int objectid = (Integer)toDataLsid.get("objectid"); + if (seen.add(objectid)) + prepEdgeForInsert(params, runObjectId, objectid, runId); + } + + for (Map toMaterialLsid : toMaterialLsids) + { + int objectid = (Integer)toMaterialLsid.get("objectid"); + if (seen.add(objectid)) + prepEdgeForInsert(params, runObjectId, objectid, runId); + } + + if (!provenanceFinalOutputs.isEmpty()) + { + for (Pair pair : provenanceFinalOutputs) + { + Integer toObjectId = pair.second; + if (null != toObjectId) + { + if (seen.add(toObjectId)) + prepEdgeForInsert(params, runObjectId, toObjectId, runId); + } + } + } + + if (verifyEdgesNoInsert) + verifyEdges(runId, runObjectId, params); + else + { + insertEdges(params); + + } + } else - insertEdges(params); - } - else - { - if (verifyEdgesNoInsert) - verifyEdges(runId, runObjectId, Collections.emptyList()); - } + { + if (verifyEdgesNoInsert) + verifyEdges(runId, runObjectId, Collections.emptyList()); + } - tx.commit(); - timer.stop(); - LOG.debug(" " + (verifyEdgesNoInsert ? "verified" : "synced") + " edges in " + timer.getDuration()); + tx.commit(); + timer.stop(); + LOG.debug(" " + (verifyEdgesNoInsert ? "verified" : "synced") + " edges in " + timer.getDuration()); + + if (!verifyEdgesNoInsert && invalidateClosureCache) + { + toCpasTypes.forEach(type -> ClosureQueryHelper.invalidateMaterialsForRun(type, runId)); + } + } } } @@ -3191,7 +3254,11 @@ public void rebuildAllEdges() String runLsid = (String)run.get("lsid"); String containerId = (String)run.get("container"); Container runContainer = ContainerManager.getForId(containerId); - syncRunEdges(runId, runObjectId, runLsid, runContainer, false, false, cpasTypeToObjectId); + new SyncRunEdges(runId, runObjectId, runLsid, runContainer) + .deleteFirst(false) + .verifyEdgesNoInsert(false) + .invalidateClosureCache(false) // don't do incremental invalidation calls + .sync(cpasTypeToObjectId); } } @@ -3201,13 +3268,19 @@ public void rebuildAllEdges() LOG.debug("Rebuilt all edges: " + timing.getDuration() + " ms"); } } + ClosureQueryHelper.invalidateAll(); } + public void verifyRunEdges(ExpRun run) { - syncRunEdges(run.getRowId(), run.getObjectId(), run.getLSID(), run.getContainer(), false, true, null); + new SyncRunEdges(run) + .deleteFirst(false) + .verifyEdgesNoInsert(true) + .sync(null); } + public void verifyAllEdges(Container c, @Nullable Integer limit) { if (c.isRoot()) @@ -3248,7 +3321,10 @@ private void _verifyAllEdges(Container c, @Nullable Integer limit) String runLsid = (String)run.get("lsid"); String containerId = (String)run.get("container"); Container runContainer = ContainerManager.getForId(containerId); - syncRunEdges(runId, runObjectId, runLsid, runContainer, false, true, cpasTypeToObjectId); + new SyncRunEdges(runId, runObjectId, runLsid, runContainer) + .deleteFirst(false) + .verifyEdgesNoInsert(true) + .sync(cpasTypeToObjectId); runCount++; if (runCount % 1000 == 0) @@ -4593,7 +4669,6 @@ WHERE m3.rowId in (3592, 3593, 3594) .append(")"); return ExpRunImpl.fromRuns(getRunsForRunIds(sql)); - } private Collection getDerivedRunsFromMaterial(Collection materialIds) @@ -6460,7 +6535,10 @@ public void saveBatch() throws SQLException, XarFormatException, ValidationExcep Map cpasTypeToObjectId = new HashMap<>(); for (var er : runLsidToRowId.values()) { - syncRunEdges(er.getRowId(), er.getObjectId(), er.getLSID(), _container, false, false, cpasTypeToObjectId); + new SyncRunEdges(er.getExpObject()) + .deleteFirst(false) + .verifyEdgesNoInsert(false) + .sync(cpasTypeToObjectId); } }