From 9b5e47ae2bbb061cfba18b74fa1c4b4534e75386 Mon Sep 17 00:00:00 2001 From: Matthew Bellew Date: Thu, 10 Mar 2022 16:04:18 -0800 Subject: [PATCH 1/9] checkpoint - prototype code for discussion --- .../experiment/api/ClosureQueryHelper.java | 113 ++++++++++++++++++ .../experiment/api/ExpMaterialTableImpl.java | 66 ++++++++++ 2 files changed, 179 insertions(+) create mode 100644 experiment/src/org/labkey/experiment/api/ClosureQueryHelper.java diff --git a/experiment/src/org/labkey/experiment/api/ClosureQueryHelper.java b/experiment/src/org/labkey/experiment/api/ClosureQueryHelper.java new file mode 100644 index 00000000000..745affff275 --- /dev/null +++ b/experiment/src/org/labkey/experiment/api/ClosureQueryHelper.java @@ -0,0 +1,113 @@ +package org.labkey.experiment.api; + +import org.labkey.api.data.ColumnInfo; +import org.labkey.api.data.DbSchema; +import org.labkey.api.data.JdbcType; +import org.labkey.api.data.MaterializedQueryHelper; +import org.labkey.api.data.MutableColumnInfo; +import org.labkey.api.data.SQLFragment; +import org.labkey.api.data.TableInfo; +import org.labkey.api.exp.api.ExpDataClass; +import org.labkey.api.exp.api.ExpSampleType; +import org.labkey.api.query.ExprColumn; +import org.labkey.api.query.QueryForeignKey; + +import java.util.Collections; +import java.util.HashMap; +import java.util.Map; + +public class ClosureQueryHelper +{ + /* TODO/CONSIDER every SampleType and Dataclass should have a unique ObjectId so it can be stored as an in lineage tables (e.g. edge/closure tables) */ + + static Map queryHelpers = Collections.synchronizedMap(new HashMap<>()); + + static String pgMaterialClosureSql = """ + WITH RECURSIVE CTE_ AS ( + + SELECT + RowId AS Start_, + ObjectId as End_, + '/' || CAST(ObjectId AS VARCHAR) || '/' as Path_, + 0 as Depth_ + FROM exp.Material + WHERE Material.cpasType = ? + + UNION ALL + + SELECT CTE_.Start_, Edge.FromObjectId as End_, CTE_.Path_ || CAST(Edge.FromObjectId AS VARCHAR) || '/' as Path_, Depth_ + 1 as Depth_ + FROM CTE_ INNER JOIN exp.Edge ON CTE_.End_ = Edge.ToObjectId + WHERE Depth_ < 100 AND 0 = POSITION('/' || CAST(Edge.FromObjectId AS VARCHAR) || '/' IN Path_) + + ) + + SELECT Start_, CASE WHEN COUNT(*) = 1 THEN MIN(rowId) ELSE -1 * COUNT(*) END AS rowId, targetId + FROM ( + SELECT Start_, End_, + COALESCE(material.rowid, dataclass.rowid) as rowId, + COALESCE('m' || CAST(materialsource.rowid AS VARCHAR), 'd' || CAST(dataclass.rowid AS VARCHAR)) as targetId + FROM CTE_ + LEFT OUTER JOIN exp.material ON End_ = material.objectId LEFT OUTER JOIN exp.materialsource ON material.cpasType = materialsource.lsid + LEFT OUTER JOIN exp.data on End_ = data.objectId LEFT OUTER JOIN exp.dataclass ON data.cpasType = dataclass.lsid + WHERE Depth_ > 0 AND materialsource.rowid IS NOT NULL OR dataclass.rowid IS NOT NULL) _inner_ + GROUP BY targetId, Start_ + """; + + /* + * THIS IS A TERRIBLE INVALIDATION QUERY. a) we don't care about deletes b) we don't even care about inserts/updates. Only + * lineage changes. Probably best to use the in-memory invalidator counter strategy. + */ + static String pgUpToDateMaterialSql = """ + SELECT CAST(MAX(rowId) AS VARCHAR) || '/' || CAST(MAX(modified) AS VARCHAR) FROM exp.material WHERE cpasType = ? + """; + + + /* + * This can be used to add a column directly to a exp table, or to create a column + * in an intermediate fake lookup table + */ + static MutableColumnInfo createLineageLookupColumn(final ColumnInfo fkRowId, ExpSampleType source, ExpSampleType target) + { + TableInfo parentTable = fkRowId.getParentTable(); + var ret = new ExprColumn(parentTable, target.getLSID(), new SQLFragment("#ERROR#"), JdbcType.INTEGER) + { + @Override + public SQLFragment getValueSql(String tableAlias) + { + SQLFragment objectId = fkRowId.getValueSql(tableAlias); + return ClosureQueryHelper.getValueSql(source.getLSID(), objectId, target); + } + }; + ret.setLabel(target.getName()); + ret.setFk(new QueryForeignKey.Builder(parentTable.getUserSchema(), parentTable.getContainerFilter()).table(target.getName())); + return ret; + } + + + public static SQLFragment getValueSql(String sourceLSID, SQLFragment objectId, ExpSampleType target) + { + return getValueSql(sourceLSID, objectId, "m" + target.getRowId()); + } + + public static SQLFragment getValueSql(String sourceLSID, SQLFragment objectId, ExpDataClass target) + { + return getValueSql(sourceLSID, objectId, "d" + target.getRowId()); + } + + private static SQLFragment getValueSql(String sourceLSID, SQLFragment objectId, String targetId) + { + /* TODO might need/want an in-memory invalidator flag/autoincrement per container/sampletype. pgUpToDateSql is good enough for prototyping */ + MaterializedQueryHelper helper = queryHelpers.computeIfAbsent(sourceLSID, cpasType -> + new MaterializedQueryHelper.Builder("closure", DbSchema.getTemp().getScope(), new SQLFragment(pgMaterialClosureSql, cpasType)) + .addIndex("CREATE UNIQUE INDEX uq_${NAME} ON temp.${NAME} (targetId,Start_)") + .upToDateSql(new SQLFragment(pgUpToDateMaterialSql, cpasType)) + .build()); + + return new SQLFragment() + .append("(SELECT rowId FROM ") + .append(helper.getFromSql("CLOS", null)) + .append(" WHERE targetId='").append(targetId).append("'") + .append(" AND Start_=").append(objectId) + .append(")"); + } +} diff --git a/experiment/src/org/labkey/experiment/api/ExpMaterialTableImpl.java b/experiment/src/org/labkey/experiment/api/ExpMaterialTableImpl.java index 4da5d3cacdc..79efb9f37fe 100644 --- a/experiment/src/org/labkey/experiment/api/ExpMaterialTableImpl.java +++ b/experiment/src/org/labkey/experiment/api/ExpMaterialTableImpl.java @@ -670,8 +670,74 @@ protected ContainerFilter getLookupContainerFilter() setTitleColumn(Column.Name.toString()); setDefaultVisibleColumns(defaultCols); + + /* ADD SAMPLETYPE LINEAGE LOOKUP COLUMNS + * + * TODO + * which SAMPLETYPE tables do we add? + * do we want an intermediate fake lookup column? + * do we want to enforce uniqueness of lookup before creating lookup column? how? + * + * Probably best to add intermediate lookup column, to avoid TableSelector(ALL_COLUMNS) doing all this work. + * See SampleTypeUpdateServiceDI.getMaterialMap(). + */ + if (null != _ss) + { + var sts = SampleTypeServiceImpl.get().getSampleTypes(_userSchema.getContainer(), _userSchema.getUser(), false); + if (null != sts && !sts.isEmpty()) + { + MutableColumnInfo wrappedRowId = wrapColumn("LineageLookupPlaceHolder", _rootTable.getColumn("rowid")); + wrappedRowId.setIsUnselectable(true); + wrappedRowId.setFk(new AbstractForeignKey(getUserSchema(),null) + { + @Override + public @Nullable ColumnInfo createLookupColumn(ColumnInfo parent, String displayField) + { + var target = SampleTypeServiceImpl.get().getSampleTypeByType(displayField, _userSchema.getContainer()); + if (null == target) + target = SampleTypeServiceImpl.get().getSampleType(_userSchema.getContainer(), _userSchema.getUser(), displayField); + if (null == target) + return null; + return ClosureQueryHelper.createLineageLookupColumn(parent, _ss, target); + } + + @Override + public @Nullable TableInfo getLookupTableInfo() + { + return new PlaceHolderLineageLookupTableInfo(); + } + + @Override + public StringExpression getURL(ColumnInfo parent) + { + return null; + } + }); + addColumn(wrappedRowId); + } + } } + + private class PlaceHolderLineageLookupTableInfo extends VirtualTable + { + PlaceHolderLineageLookupTableInfo() + { + super(ExpMaterialTableImpl.this.getSchema(), "LineageLookupPlaceHolder", ExpMaterialTableImpl.this.getUserSchema()); + List sts = SampleTypeServiceImpl.get().getSampleTypes(_userSchema.getContainer(), _userSchema.getUser(), false); + ColumnInfo wrap = new BaseColumnInfo("rowid", this, JdbcType.INTEGER); + for (ExpSampleTypeImpl target : sts) + addColumn(ClosureQueryHelper.createLineageLookupColumn(wrap, _ss, target)); + } + + @Override + public @NotNull SQLFragment getFromSQL() + { + throw new IllegalStateException(); + } + } + + @Override public Domain getDomain() { From 2f36d8657e084fa86e00b13bed23ce7d9d59c1ca Mon Sep 17 00:00:00 2001 From: Matthew Bellew Date: Mon, 14 Mar 2022 15:18:13 -0700 Subject: [PATCH 2/9] lower max depth for lineage lookup closure query --- .../api/data/MaterializedQueryHelper.java | 110 ++++++++----- .../org/labkey/api/data/TableSelector.java | 5 +- .../experiment/api/ClosureQueryHelper.java | 146 +++++++++++++++--- .../experiment/api/ExpMaterialTableImpl.java | 3 + .../experiment/api/ExperimentServiceImpl.java | 18 ++- 5 files changed, 222 insertions(+), 60 deletions(-) diff --git a/api/src/org/labkey/api/data/MaterializedQueryHelper.java b/api/src/org/labkey/api/data/MaterializedQueryHelper.java index 4750e55189a..cf9ea32cedf 100644 --- a/api/src/org/labkey/api/data/MaterializedQueryHelper.java +++ b/api/src/org/labkey/api/data/MaterializedQueryHelper.java @@ -56,12 +56,14 @@ private class Materialized private final long _created; private final String _cacheKey; private final String _fromSql; + private final String _tableName; private final ArrayList _invalidators = new ArrayList<>(3); - Materialized(String cacheKey, long created, String sql) + Materialized(String tableName, String cacheKey, long created, String sql) { _created = created; _cacheKey = cacheKey; + _tableName = tableName; _fromSql = sql; } @@ -306,54 +308,41 @@ public SQLFragment getFromSql(String tableAlias, Container c) return getFromSql(_selectQuery, tableAlias, c); } + + public boolean isCached(Container c) + { + if (null == _selectQuery) + throw new IllegalStateException("Must specify source query in constructor or in getFromSql()"); + return null != getMaterialized(makeKey(_scope.getCurrentTransaction(), c)); + } + + + public void upsert(SQLFragment sqlf) + { + String txCacheKey = makeKey(_scope.getCurrentTransaction(), null); + Materialized m = getMaterialized(txCacheKey); + if (null == m) + return; + String sql = sqlf.getSQL().replace("${NAME}", m._tableName); + List params = sqlf.getParams(); + new SqlExecutor(_scope).execute(new SQLFragment(sql,params)); + } + + /* NOTE: we do not want to hold synchronized(this) while doing any SQL operations */ public SQLFragment getFromSql(@NotNull SQLFragment selectQuery, String tableAlias, Container c) { - Materialized materialized = null; final String txCacheKey = makeKey(_scope.getCurrentTransaction(), c); final long now = HeartBeat.currentTimeMillis(); - synchronized (this) - { - if (_closed) - throw new IllegalStateException(); - if (null != c) - throw new UnsupportedOperationException(); - - _countGetFromSql.incrementAndGet(); - - if (_scope.isTransactionActive()) - materialized = _map.get(txCacheKey); - - if (null == materialized) - materialized = _map.get(makeKey(null, c)); - } - - if (null != materialized) - { - boolean replace = false; - for (Invalidator i : materialized._invalidators) - { - CacheCheck cc = i.checkValid(materialized._created); - if (cc != CacheCheck.OK) - replace = true; - } - if (replace) - { - synchronized (this) - { - _map.remove(materialized._cacheKey); - materialized = null; - } - } - } + Materialized materialized = getMaterialized(txCacheKey); if (null == materialized) { _countSelectInto.incrementAndGet(); DbSchema temp = DbSchema.getTemp(); String name = _prefix + "_" + GUID.makeHash(); - materialized = new Materialized(txCacheKey, now, "\"" + temp.getName() + "\".\"" + name + "\""); + materialized = new Materialized(name, txCacheKey, now, "\"" + temp.getName() + "\".\"" + name + "\""); materialized.addMaxTimeToCache(_maxTimeToCache); materialized.addUpToDateQuery(_uptodateQuery); materialized.addInvalidator(_supplier); @@ -394,6 +383,53 @@ public SQLFragment getFromSql(@NotNull SQLFragment selectQuery, String tableAlia return sqlf; } + @Nullable + private Materialized getMaterialized(String txCacheKey) + { + Materialized materialized = null; + + synchronized (this) + { + if (_closed) + throw new IllegalStateException(); + + _countGetFromSql.incrementAndGet(); + + if (_scope.isTransactionActive()) + materialized = _map.get(txCacheKey); + + if (null == materialized) + materialized = _map.get(makeKey(null, null)); + } + + if (null != materialized) + { + boolean replace = false; + for (Invalidator i : materialized._invalidators) + { + CacheCheck cc = i.checkValid(materialized._created); + if (cc != CacheCheck.OK) + replace = true; + } + if (replace) + { + synchronized (this) + { + _map.remove(materialized._cacheKey); + materialized = null; + } + } + } + return materialized; + } + + + /* Do incremental update to existing cached data. There is no provision for deleting rows. */ + public void upsert() + { + + } + /** * To be consistent with CacheManager maxTimeToCache==0 means UNLIMITED, so we use maxTimeToCache==-1 to mean no caching, just materialize and return diff --git a/api/src/org/labkey/api/data/TableSelector.java b/api/src/org/labkey/api/data/TableSelector.java index d3e8ad3648b..f66e224ec74 100644 --- a/api/src/org/labkey/api/data/TableSelector.java +++ b/api/src/org/labkey/api/data/TableSelector.java @@ -42,6 +42,7 @@ import java.util.List; import java.util.Map; import java.util.Set; +import java.util.stream.Collectors; import java.util.stream.Stream; public class TableSelector extends SqlExecutingSelector implements ResultsFactory @@ -129,7 +130,9 @@ private static Collection columnInfosList(@NotNull TableInfo table, if (select == ALL_COLUMNS) { - selectColumns = table.getColumns(); + selectColumns = table.getColumns().stream() + .filter(columnInfo -> !columnInfo.isUnselectable()) + .collect(Collectors.toList()); } else { diff --git a/experiment/src/org/labkey/experiment/api/ClosureQueryHelper.java b/experiment/src/org/labkey/experiment/api/ClosureQueryHelper.java index 745affff275..836810fdd41 100644 --- a/experiment/src/org/labkey/experiment/api/ClosureQueryHelper.java +++ b/experiment/src/org/labkey/experiment/api/ClosureQueryHelper.java @@ -1,7 +1,9 @@ package org.labkey.experiment.api; import org.labkey.api.data.ColumnInfo; +import org.labkey.api.data.CoreSchema; import org.labkey.api.data.DbSchema; +import org.labkey.api.data.DbScope; import org.labkey.api.data.JdbcType; import org.labkey.api.data.MaterializedQueryHelper; import org.labkey.api.data.MutableColumnInfo; @@ -15,14 +17,20 @@ import java.util.Collections; import java.util.HashMap; import java.util.Map; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicInteger; + public class ClosureQueryHelper { /* TODO/CONSIDER every SampleType and Dataclass should have a unique ObjectId so it can be stored as an in lineage tables (e.g. edge/closure tables) */ - static Map queryHelpers = Collections.synchronizedMap(new HashMap<>()); + static final Map queryHelpers = Collections.synchronizedMap(new HashMap<>()); + static final Map invalidators = Collections.synchronizedMap(new HashMap<>()); + + static final int MAX_LINEAGE_LOOKUP_DEPTH = 10; - static String pgMaterialClosureSql = """ + static String pgMaterialClosureSql = String.format(""" WITH RECURSIVE CTE_ AS ( SELECT @@ -30,36 +38,28 @@ WITH RECURSIVE CTE_ AS ( ObjectId as End_, '/' || CAST(ObjectId AS VARCHAR) || '/' as Path_, 0 as Depth_ - FROM exp.Material - WHERE Material.cpasType = ? + /*FROM*/ UNION ALL SELECT CTE_.Start_, Edge.FromObjectId as End_, CTE_.Path_ || CAST(Edge.FromObjectId AS VARCHAR) || '/' as Path_, Depth_ + 1 as Depth_ FROM CTE_ INNER JOIN exp.Edge ON CTE_.End_ = Edge.ToObjectId - WHERE Depth_ < 100 AND 0 = POSITION('/' || CAST(Edge.FromObjectId AS VARCHAR) || '/' IN Path_) + WHERE Depth_ < %d AND 0 = POSITION('/' || CAST(Edge.FromObjectId AS VARCHAR) || '/' IN Path_) ) SELECT Start_, CASE WHEN COUNT(*) = 1 THEN MIN(rowId) ELSE -1 * COUNT(*) END AS rowId, targetId + /* INTO */ FROM ( SELECT Start_, End_, - COALESCE(material.rowid, dataclass.rowid) as rowId, + COALESCE(material.rowid, dataclass.rowid) as rowId, COALESCE('m' || CAST(materialsource.rowid AS VARCHAR), 'd' || CAST(dataclass.rowid AS VARCHAR)) as targetId FROM CTE_ LEFT OUTER JOIN exp.material ON End_ = material.objectId LEFT OUTER JOIN exp.materialsource ON material.cpasType = materialsource.lsid LEFT OUTER JOIN exp.data on End_ = data.objectId LEFT OUTER JOIN exp.dataclass ON data.cpasType = dataclass.lsid WHERE Depth_ > 0 AND materialsource.rowid IS NOT NULL OR dataclass.rowid IS NOT NULL) _inner_ GROUP BY targetId, Start_ - """; - - /* - * THIS IS A TERRIBLE INVALIDATION QUERY. a) we don't care about deletes b) we don't even care about inserts/updates. Only - * lineage changes. Probably best to use the in-memory invalidator counter strategy. - */ - static String pgUpToDateMaterialSql = """ - SELECT CAST(MAX(rowId) AS VARCHAR) || '/' || CAST(MAX(modified) AS VARCHAR) FROM exp.material WHERE cpasType = ? - """; + """, MAX_LINEAGE_LOOKUP_DEPTH); /* @@ -89,19 +89,16 @@ public static SQLFragment getValueSql(String sourceLSID, SQLFragment objectId, E return getValueSql(sourceLSID, objectId, "m" + target.getRowId()); } + public static SQLFragment getValueSql(String sourceLSID, SQLFragment objectId, ExpDataClass target) { return getValueSql(sourceLSID, objectId, "d" + target.getRowId()); } + private static SQLFragment getValueSql(String sourceLSID, SQLFragment objectId, String targetId) { - /* TODO might need/want an in-memory invalidator flag/autoincrement per container/sampletype. pgUpToDateSql is good enough for prototyping */ - MaterializedQueryHelper helper = queryHelpers.computeIfAbsent(sourceLSID, cpasType -> - new MaterializedQueryHelper.Builder("closure", DbSchema.getTemp().getScope(), new SQLFragment(pgMaterialClosureSql, cpasType)) - .addIndex("CREATE UNIQUE INDEX uq_${NAME} ON temp.${NAME} (targetId,Start_)") - .upToDateSql(new SQLFragment(pgUpToDateMaterialSql, cpasType)) - .build()); + MaterializedQueryHelper helper = getClosureHelper(sourceLSID); return new SQLFragment() .append("(SELECT rowId FROM ") @@ -110,4 +107,111 @@ private static SQLFragment getValueSql(String sourceLSID, SQLFragment objectId, .append(" AND Start_=").append(objectId) .append(")"); } + + +/* + public static void recomputeLineage(String sourceLSID, Collection> rowid_objectids) + { + if (rowid_objectids.isEmpty()) + return; + + if (getScope().isTransactionActive()) + { + // TODO/CONSIDER handle the tx case? + getInvalidationCounter(sourceLSID).incrementAndGet(); + return; + } + + // if there's nothing cached, we don't need to do incremental + MaterializedQueryHelper helper = getClosureHelper(sourceLSID); + if (!helper.isCached(null)) + return; + + TempTableTracker ttt = null; + + try + { + // COMPUTE closure for given rows, save into temp table + StringBuilder from = new StringBuilder(" FROM (VALUES "); + String comma = ""; + for (var p : rowid_objectids) + { + from.append(comma); + from.append("(").append(p.first).append(",").append(p.second).append(")"); + comma = ",\n"; + } + from.append(") AS _mat_ (RowId,ObjectId) "); + String sql = pgMaterialClosureSql.replace("/*FROM*\/", from); + Object ref = new Object(); + String tempTableName = "closure"+temptableNumber.incrementAndGet(); + ttt = TempTableTracker.track("closure"+temptableNumber.incrementAndGet(), ref); + SQLFragment selectInto = new SQLFragment("SELECT * INTO temp.\"" + tempTableName + "\"\nFROM (\n") + .append(new SQLFragment(sql)) + .append("\n) _sql_"); + new SqlExecutor(getScope()).execute(selectInto); + + SQLFragment upsert = new SQLFragment() + .append("INSERT INTO temp.${NAME} (Start_, rowId, targetid)\n").append("SELECT Start, RowId, targetId FROM temp.").append(tempTableName).append(" TMP\n") + .append("ON CONFLICT(Start_,targetId) UPDATE SET rowId = EXCLUDED.rowId"); + + helper.upsert(upsert); + return; + } + catch (Exception x) + { + getInvalidationCounter(sourceLSID).incrementAndGet(); + throw x; + } + finally + { + if (null != ttt) + ttt.delete(); + } + } +*/ + + public static void invalidateAll() + { + synchronized (queryHelpers) + { + for (var h : queryHelpers.values()) + h.uncache(null); + } + } + + + public static void invalidateCpasType(String lsid) + { + var counter = invalidators.get(lsid); + if (null != counter) + counter.incrementAndGet(); + } + + + private static MaterializedQueryHelper getClosureHelper(String sourceLSID) + { + return queryHelpers.computeIfAbsent(sourceLSID, cpasType -> + { + String sql = pgMaterialClosureSql.replace("/*FROM*/", "FROM exp.Material WHERE Material.cpasType = ?"); + return new MaterializedQueryHelper.Builder("closure", DbSchema.getTemp().getScope(), new SQLFragment(sql, cpasType)) + .addIndex("CREATE UNIQUE INDEX uq_${NAME} ON temp.${NAME} (targetId,Start_)") + .maxTimeToCache(TimeUnit.MINUTES.toMillis(5)) + .addInvalidCheck(() -> String.valueOf(getInvalidationCounter(sourceLSID))) + .build(); + }); + } + + + static final AtomicInteger temptableNumber = new AtomicInteger(); + + private static AtomicInteger getInvalidationCounter(String sourceLSID) + { + return invalidators.computeIfAbsent(sourceLSID, (key) -> new AtomicInteger()); + } + + + private static DbScope getScope() + { + return CoreSchema.getInstance().getScope(); + } } diff --git a/experiment/src/org/labkey/experiment/api/ExpMaterialTableImpl.java b/experiment/src/org/labkey/experiment/api/ExpMaterialTableImpl.java index 79efb9f37fe..9c64ad4339a 100644 --- a/experiment/src/org/labkey/experiment/api/ExpMaterialTableImpl.java +++ b/experiment/src/org/labkey/experiment/api/ExpMaterialTableImpl.java @@ -688,6 +688,9 @@ protected ContainerFilter getLookupContainerFilter() { MutableColumnInfo wrappedRowId = wrapColumn("LineageLookupPlaceHolder", _rootTable.getColumn("rowid")); wrappedRowId.setIsUnselectable(true); + wrappedRowId.setReadOnly(true); + wrappedRowId.setCalculated(true); + wrappedRowId.setRequired(false); wrappedRowId.setFk(new AbstractForeignKey(getUserSchema(),null) { @Override diff --git a/experiment/src/org/labkey/experiment/api/ExperimentServiceImpl.java b/experiment/src/org/labkey/experiment/api/ExperimentServiceImpl.java index 12b6e8a835b..80e8c521274 100644 --- a/experiment/src/org/labkey/experiment/api/ExperimentServiceImpl.java +++ b/experiment/src/org/labkey/experiment/api/ExperimentServiceImpl.java @@ -3020,13 +3020,20 @@ private void syncRunEdges(int runId, Integer runObjectId, String runLsid, Contai .append("INNER JOIN exp.ProtocolApplication pa ON mi.TargetApplicationId = pa.RowId\n") .append("WHERE pa.RunId = ").append(runId).append(" AND pa.CpasType IN ('").append(ExperimentRun.name()).append("','").append(ExperimentRunOutput.name()).append("')"); + Set fromCpasTypes = new HashSet<>(); Collection> fromMaterialLsids = new ArrayList<>(); Collection> toMaterialLsids = new ArrayList<>(); new SqlSelector(getSchema(), materials).forEachMap(row -> { - if (ExperimentRun.name().equals(row.get("pa_cpas_type"))) + String cpasType = (String)row.get("pa_cpas_type"); + if (ExperimentRun.name().equals(cpasType)) + { + fromCpasTypes.add(cpasType); fromMaterialLsids.add(row); + } else + { toMaterialLsids.add(row); + } }); Set> provenanceStartingInputs = emptySet(); @@ -3151,7 +3158,10 @@ private void syncRunEdges(int runId, Integer runObjectId, String runLsid, Contai if (verifyEdgesNoInsert) verifyEdges(runId, runObjectId, params); else + { insertEdges(params); + + } } else { @@ -3163,6 +3173,11 @@ private void syncRunEdges(int runId, Integer runObjectId, String runLsid, Contai tx.commit(); timer.stop(); LOG.debug(" " + (verifyEdgesNoInsert ? "verified" : "synced") + " edges in " + timer.getDuration()); + + if (!verifyEdgesNoInsert && !fromCpasTypes.isEmpty()) + { + fromCpasTypes.forEach((type) -> ClosureQueryHelper.invalidateCpasType(type)); + } } } @@ -3201,6 +3216,7 @@ public void rebuildAllEdges() LOG.debug("Rebuilt all edges: " + timing.getDuration() + " ms"); } } + ClosureQueryHelper.invalidateAll(); } public void verifyRunEdges(ExpRun run) From 10fa9fe896b185a5f25a3c438a7a9a88c4c2d068 Mon Sep 17 00:00:00 2001 From: Matthew Bellew Date: Tue, 15 Mar 2022 14:01:43 -0700 Subject: [PATCH 3/9] Sql server invalidateMaterialsForRun() --- .../api/data/MaterializedQueryHelper.java | 44 +- .../experiment/api/ClosureQueryHelper.java | 168 ++++++-- .../experiment/api/ExperimentServiceImpl.java | 403 ++++++++++-------- 3 files changed, 390 insertions(+), 225 deletions(-) diff --git a/api/src/org/labkey/api/data/MaterializedQueryHelper.java b/api/src/org/labkey/api/data/MaterializedQueryHelper.java index cf9ea32cedf..425e063787d 100644 --- a/api/src/org/labkey/api/data/MaterializedQueryHelper.java +++ b/api/src/org/labkey/api/data/MaterializedQueryHelper.java @@ -193,11 +193,11 @@ private String makeKey(DbScope.Transaction t, Container c) private final String _prefix; private final DbScope _scope; private final SQLFragment _selectQuery; + private final boolean _isSelectIntoSql; private final SQLFragment _uptodateQuery; private final Supplier _supplier; private final List _indexes = new ArrayList<>(); private final long _maxTimeToCache; - private final boolean _perContainer; private final LinkedHashMap _map = new LinkedHashMap() { @Override @@ -216,7 +216,7 @@ protected boolean removeEldestEntry(Map.Entry eldest) private boolean _closed = false; private MaterializedQueryHelper(String prefix, DbScope scope, SQLFragment select, @Nullable SQLFragment uptodate, Supplier supplier, @Nullable Collection indexes, long maxTimeToCache, - boolean perContainer) + boolean isSelectIntoSql) { _prefix = StringUtils.defaultString(prefix,"mat"); _scope = scope; @@ -224,11 +224,9 @@ private MaterializedQueryHelper(String prefix, DbScope scope, SQLFragment select _uptodateQuery = uptodate; _supplier = supplier; _maxTimeToCache = maxTimeToCache; - _perContainer = perContainer; if (null != indexes) _indexes.addAll(indexes); - if (perContainer) - throw new UnsupportedOperationException("NYI"); + _isSelectIntoSql = isSelectIntoSql; assert MemTracker.get().put(this); } @@ -305,7 +303,7 @@ public SQLFragment getFromSql(String tableAlias, Container c) { if (null == _selectQuery) throw new IllegalStateException("Must specify source query in constructor or in getFromSql()"); - return getFromSql(_selectQuery, tableAlias, c); + return getFromSql(_selectQuery, _isSelectIntoSql, tableAlias, c); } @@ -329,8 +327,14 @@ public void upsert(SQLFragment sqlf) } - /* NOTE: we do not want to hold synchronized(this) while doing any SQL operations */ + /* used by FLow directly for some reason */ public SQLFragment getFromSql(@NotNull SQLFragment selectQuery, String tableAlias, Container c) + { + return getFromSql(selectQuery, false, tableAlias, c); + } + + /* NOTE: we do not want to hold synchronized(this) while doing any SQL operations */ + public SQLFragment getFromSql(@NotNull SQLFragment selectQuery, boolean isSelectInto, String tableAlias, Container c) { final String txCacheKey = makeKey(_scope.getCurrentTransaction(), c); final long now = HeartBeat.currentTimeMillis(); @@ -351,9 +355,19 @@ public SQLFragment getFromSql(@NotNull SQLFragment selectQuery, String tableAlia TempTableTracker.track(name, materialized); - SQLFragment selectInto = new SQLFragment("SELECT * INTO \"" + temp.getName() + "\".\"" + name + "\"\nFROM (\n"); - selectInto.append(selectQuery); - selectInto.append("\n) _sql_"); + SQLFragment selectInto; + if (isSelectInto) + { + String sql = selectQuery.getSQL().replace("${NAME}", name); + List params = selectQuery.getParams(); + selectInto = new SQLFragment(sql,params); + } + else + { + selectInto = new SQLFragment("SELECT * INTO \"" + temp.getName() + "\".\"" + name + "\"\nFROM (\n"); + selectInto.append(selectQuery); + selectInto.append("\n) _sql_"); + } new SqlExecutor(_scope).execute(selectInto); try (var ignored = SpringActionController.ignoreSqlUpdates()) @@ -463,6 +477,7 @@ public static class Builder implements org.labkey.api.data.Builder _supplier = null; @@ -475,6 +490,13 @@ public Builder(String prefix, DbScope scope, SQLFragment select) _select = select; } + /** This property indicates that the SQLFragment is formatted as a SELECT INTO query (rather than a simple SELECT) */ + public Builder setIsSelectInto(boolean b) + { + _isSelectInto = b; + return this; + } + public Builder upToDateSql(SQLFragment uptodate) { _uptodate = uptodate; @@ -502,7 +524,7 @@ public Builder addIndex(String index) @Override public MaterializedQueryHelper build() { - return new MaterializedQueryHelper(_prefix, _scope, _select, _uptodate, _supplier, _indexes, _max, false); + return new MaterializedQueryHelper(_prefix, _scope, _select, _uptodate, _supplier, _indexes, _max, _isSelectInto); } } diff --git a/experiment/src/org/labkey/experiment/api/ClosureQueryHelper.java b/experiment/src/org/labkey/experiment/api/ClosureQueryHelper.java index 836810fdd41..a643f467996 100644 --- a/experiment/src/org/labkey/experiment/api/ClosureQueryHelper.java +++ b/experiment/src/org/labkey/experiment/api/ClosureQueryHelper.java @@ -1,5 +1,7 @@ package org.labkey.experiment.api; +import org.apache.commons.lang3.StringUtils; +import org.jetbrains.annotations.Nullable; import org.labkey.api.data.ColumnInfo; import org.labkey.api.data.CoreSchema; import org.labkey.api.data.DbSchema; @@ -8,18 +10,25 @@ import org.labkey.api.data.MaterializedQueryHelper; import org.labkey.api.data.MutableColumnInfo; import org.labkey.api.data.SQLFragment; +import org.labkey.api.data.SqlExecutor; import org.labkey.api.data.TableInfo; +import org.labkey.api.data.TempTableTracker; +import org.labkey.api.data.dialect.SqlDialect; import org.labkey.api.exp.api.ExpDataClass; import org.labkey.api.exp.api.ExpSampleType; import org.labkey.api.query.ExprColumn; import org.labkey.api.query.QueryForeignKey; +import org.labkey.api.query.UserSchema; import java.util.Collections; import java.util.HashMap; import java.util.Map; +import java.util.Objects; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicInteger; +import static org.labkey.api.exp.api.ExpProtocol.ApplicationType.ExperimentRunOutput; + public class ClosureQueryHelper { @@ -30,7 +39,7 @@ public class ClosureQueryHelper static final int MAX_LINEAGE_LOOKUP_DEPTH = 10; - static String pgMaterialClosureSql = String.format(""" + static String pgMaterialClosureCTE = String.format(""" WITH RECURSIVE CTE_ AS ( SELECT @@ -45,9 +54,11 @@ WITH RECURSIVE CTE_ AS ( SELECT CTE_.Start_, Edge.FromObjectId as End_, CTE_.Path_ || CAST(Edge.FromObjectId AS VARCHAR) || '/' as Path_, Depth_ + 1 as Depth_ FROM CTE_ INNER JOIN exp.Edge ON CTE_.End_ = Edge.ToObjectId WHERE Depth_ < %d AND 0 = POSITION('/' || CAST(Edge.FromObjectId AS VARCHAR) || '/' IN Path_) - + ) + """, MAX_LINEAGE_LOOKUP_DEPTH); + static String pgMaterialClosureSql = String.format(""" SELECT Start_, CASE WHEN COUNT(*) = 1 THEN MIN(rowId) ELSE -1 * COUNT(*) END AS rowId, targetId /* INTO */ FROM ( @@ -62,6 +73,71 @@ SELECT Start_, CASE WHEN COUNT(*) = 1 THEN MIN(rowId) ELSE -1 * COUNT(*) END AS """, MAX_LINEAGE_LOOKUP_DEPTH); + static String mssqlMaterialClosureCTE = String.format(""" + WITH CTE_ AS ( + + SELECT + RowId AS Start_, + ObjectId as End_, + '/' + CAST(ObjectId AS VARCHAR(MAX)) + '/' as Path_, + 0 as Depth_ + /*FROM*/ + + UNION ALL + + SELECT CTE_.Start_, Edge.FromObjectId as End_, CTE_.Path_ + CAST(Edge.FromObjectId AS VARCHAR) + '/' as Path_, Depth_ + 1 as Depth_ + FROM CTE_ INNER JOIN exp.Edge ON CTE_.End_ = Edge.ToObjectId + WHERE Depth_ < %d AND 0 = CHARINDEX('/' + CAST(Edge.FromObjectId AS VARCHAR) + '/', Path_) + ) + """, (MAX_LINEAGE_LOOKUP_DEPTH)); + + static String mssqlMaterialClosureSql = """ + SELECT Start_, CASE WHEN COUNT(*) = 1 THEN MIN(rowId) ELSE -1 * COUNT(*) END AS rowId, targetId + /*INTO*/ + FROM ( + SELECT Start_, End_, + COALESCE(material.rowid, dataclass.rowid) as rowId, + COALESCE('m' + CAST(materialsource.rowid AS VARCHAR), 'd' + CAST(dataclass.rowid AS VARCHAR)) as targetId + FROM CTE_ + LEFT OUTER JOIN exp.material ON End_ = material.objectId LEFT OUTER JOIN exp.materialsource ON material.cpasType = materialsource.lsid + LEFT OUTER JOIN exp.data on End_ = data.objectId LEFT OUTER JOIN exp.dataclass ON data.cpasType = dataclass.lsid + WHERE Depth_ > 0 AND materialsource.rowid IS NOT NULL OR dataclass.rowid IS NOT NULL) _inner_ + GROUP BY targetId, Start_ + """; + + + static SQLFragment selectIntoSql(SqlDialect d, SQLFragment from, @Nullable String tempTable) + { + String cte = d.isPostgreSQL() ? pgMaterialClosureCTE : mssqlMaterialClosureCTE; + String select = d.isPostgreSQL() ? pgMaterialClosureSql : mssqlMaterialClosureSql; + + String[] cteParts = StringUtils.splitByWholeSeparator(cte,"/*FROM*/"); + assert cteParts.length == 2; + + String into = " INTO temp.${NAME} "; + if (null != tempTable) + into = " INTO temp." + tempTable + " "; + String[] selectIntoParts = StringUtils.splitByWholeSeparator(select,"/*INTO*/"); + assert selectIntoParts.length == 2; + + return new SQLFragment() + .append(cteParts[0]).append(" ").append(from).append(" ").append(cteParts[1]) + .append(selectIntoParts[0]).append(into).append(selectIntoParts[1]); + } + + + static SQLFragment selectSql(SqlDialect d, SQLFragment from) + { + String cte = d.isPostgreSQL() ? pgMaterialClosureCTE : mssqlMaterialClosureCTE; + String select = d.isPostgreSQL() ? pgMaterialClosureSql : mssqlMaterialClosureSql; + + String[] cteParts = StringUtils.splitByWholeSeparator(cte,"/*FROM*/"); + assert cteParts.length == 2; + + return new SQLFragment(cteParts[0]).append(from).append(cteParts[1]).append(select); + } + + /* * This can be used to add a column directly to a exp table, or to create a column * in an intermediate fake lookup table @@ -79,7 +155,8 @@ public SQLFragment getValueSql(String tableAlias) } }; ret.setLabel(target.getName()); - ret.setFk(new QueryForeignKey.Builder(parentTable.getUserSchema(), parentTable.getContainerFilter()).table(target.getName())); + UserSchema schema = Objects.requireNonNull(parentTable.getUserSchema()); + ret.setFk(new QueryForeignKey.Builder(schema, parentTable.getContainerFilter()).table(target.getName())); return ret; } @@ -98,7 +175,7 @@ public static SQLFragment getValueSql(String sourceLSID, SQLFragment objectId, E private static SQLFragment getValueSql(String sourceLSID, SQLFragment objectId, String targetId) { - MaterializedQueryHelper helper = getClosureHelper(sourceLSID); + MaterializedQueryHelper helper = getClosureHelper(sourceLSID, true); return new SQLFragment() .append("(SELECT rowId FROM ") @@ -109,45 +186,22 @@ private static SQLFragment getValueSql(String sourceLSID, SQLFragment objectId, } -/* - public static void recomputeLineage(String sourceLSID, Collection> rowid_objectids) - { - if (rowid_objectids.isEmpty()) - return; - - if (getScope().isTransactionActive()) - { - // TODO/CONSIDER handle the tx case? - getInvalidationCounter(sourceLSID).incrementAndGet(); - return; - } + static final AtomicInteger temptableNumber = new AtomicInteger(); + private static void incrementalRecompute(String sourceLSID, SQLFragment from) + { // if there's nothing cached, we don't need to do incremental - MaterializedQueryHelper helper = getClosureHelper(sourceLSID); - if (!helper.isCached(null)) + MaterializedQueryHelper helper = getClosureHelper(sourceLSID, false); + if (null == helper || !helper.isCached(null)) return; TempTableTracker ttt = null; - try { - // COMPUTE closure for given rows, save into temp table - StringBuilder from = new StringBuilder(" FROM (VALUES "); - String comma = ""; - for (var p : rowid_objectids) - { - from.append(comma); - from.append("(").append(p.first).append(",").append(p.second).append(")"); - comma = ",\n"; - } - from.append(") AS _mat_ (RowId,ObjectId) "); - String sql = pgMaterialClosureSql.replace("/*FROM*\/", from); Object ref = new Object(); - String tempTableName = "closure"+temptableNumber.incrementAndGet(); - ttt = TempTableTracker.track("closure"+temptableNumber.incrementAndGet(), ref); - SQLFragment selectInto = new SQLFragment("SELECT * INTO temp.\"" + tempTableName + "\"\nFROM (\n") - .append(new SQLFragment(sql)) - .append("\n) _sql_"); + String tempTableName = "closinc_"+temptableNumber.incrementAndGet(); + ttt = TempTableTracker.track(tempTableName, ref); + SQLFragment selectInto = selectIntoSql(getScope().getSqlDialect(), from, tempTableName); new SqlExecutor(getScope()).execute(selectInto); SQLFragment upsert = new SQLFragment() @@ -155,7 +209,6 @@ public static void recomputeLineage(String sourceLSID, Collection invalidateMaterialsForRun(sourceTypeLsid, runId), DbScope.CommitTaskOption.POSTCOMMIT); + return; + } + + SQLFragment seedFrom = new SQLFragment() + .append("FROM (SELECT m.RowId, m.ObjectId FROM exp.material m\n") + .append("INNER JOIN exp.MaterialInput mi ON m.rowId = mi.materialId\n") + .append("INNER JOIN exp.ProtocolApplication pa ON mi.TargetApplicationId = pa.RowId\n") + .append("WHERE pa.RunId = ").append(runId) + .append(" AND m.cpasType = ? ").add(sourceTypeLsid) + .append(" AND pa.CpasType = '").append(ExperimentRunOutput.name()).append("') _seed_ "); + incrementalRecompute(sourceTypeLsid, seedFrom); } - private static MaterializedQueryHelper getClosureHelper(String sourceLSID) + private static MaterializedQueryHelper getClosureHelper(String sourceLSID, boolean computeIfAbsent) { + if (!computeIfAbsent) + return queryHelpers.get(sourceLSID); + return queryHelpers.computeIfAbsent(sourceLSID, cpasType -> { - String sql = pgMaterialClosureSql.replace("/*FROM*/", "FROM exp.Material WHERE Material.cpasType = ?"); - return new MaterializedQueryHelper.Builder("closure", DbSchema.getTemp().getScope(), new SQLFragment(sql, cpasType)) + SQLFragment from = new SQLFragment(" FROM exp.Material WHERE Material.cpasType = ? ").add(cpasType); + SQLFragment selectInto = selectIntoSql(getScope().getSqlDialect(), from, null); + return new MaterializedQueryHelper.Builder("closure", DbSchema.getTemp().getScope(), selectInto) + .setIsSelectInto(true) .addIndex("CREATE UNIQUE INDEX uq_${NAME} ON temp.${NAME} (targetId,Start_)") .maxTimeToCache(TimeUnit.MINUTES.toMillis(5)) .addInvalidCheck(() -> String.valueOf(getInvalidationCounter(sourceLSID))) @@ -202,8 +280,6 @@ private static MaterializedQueryHelper getClosureHelper(String sourceLSID) } - static final AtomicInteger temptableNumber = new AtomicInteger(); - private static AtomicInteger getInvalidationCounter(String sourceLSID) { return invalidators.computeIfAbsent(sourceLSID, (key) -> new AtomicInteger()); diff --git a/experiment/src/org/labkey/experiment/api/ExperimentServiceImpl.java b/experiment/src/org/labkey/experiment/api/ExperimentServiceImpl.java index 80e8c521274..1cdf4daaf8f 100644 --- a/experiment/src/org/labkey/experiment/api/ExperimentServiceImpl.java +++ b/experiment/src/org/labkey/experiment/api/ExperimentServiceImpl.java @@ -2900,7 +2900,7 @@ public SyncRunEdgesTask(int runId, Integer runObjectId, String runLsid, Containe public void run() { if (_runObjectId !=null && _runLsid != null && _runContainer != null) - syncRunEdges(_runId, _runObjectId, _runLsid, _runContainer); + new SyncRunEdges(_runId, _runObjectId, _runLsid, _runContainer).sync(null); else syncRunEdges(_runId); } @@ -2958,225 +2958,278 @@ public void syncRunEdges(int runId) { ExpRun run = getExpRun(runId); if (run != null) - syncRunEdges(run); + new SyncRunEdges(run).sync(null); } + @Override public void syncRunEdges(ExpRun run) { - syncRunEdges(run.getRowId(), run.getObjectId(), run.getLSID(), run.getContainer()); + new SyncRunEdges(run).sync(null); } + @Override public void syncRunEdges(Collection runs) { + Map cpasTypeToObjectId = new HashMap<>(); + for (ExpRun run : runs) - syncRunEdges(run.getRowId(), run.getObjectId(), run.getLSID(), run.getContainer()); + { + new SyncRunEdges(run).sync(cpasTypeToObjectId); + } } - public void syncRunEdges(int runId, Integer runObjectId, String runLsid, Container runContainer) - { - syncRunEdges(runId, runObjectId, runLsid, runContainer, true, false, null); - } - private void syncRunEdges(int runId, Integer runObjectId, String runLsid, Container runContainer, boolean deleteFirst, boolean verifyEdgesNoInsert, @Nullable Map cpasTypeToObjectId) + /* syncRunEdges() has too many boolean parameters, so here's a mini builder */ + + class SyncRunEdges { - // don't do any updates if we are just verifying - if (verifyEdgesNoInsert) - deleteFirst = false; + final int _runId; + final Integer _runObjectId; + final String _runLsid; + final Container _runContainer; + boolean deleteFirst = true; + boolean verifyEdgesNoInsert=false; + boolean invalidateClosureCache=true; - CPUTimer timer = new CPUTimer("sync edges"); - timer.start(); + SyncRunEdges(ExpRun run) + { + this._runId = run.getRowId(); + this._runObjectId = run.getObjectId(); + this._runLsid = run.getLSID(); + this._runContainer = run.getContainer(); + } - LOG.debug((verifyEdgesNoInsert ? "Verifying" : "Rebuilding") + " edges for runId " + runId); - try (DbScope.Transaction tx = getExpSchema().getScope().ensureTransaction()) + SyncRunEdges(int runId, Integer runObjectId, String runLsid, Container runContainer) { - // NOTE: Originally, we just filtered exp.data by runId. This works for most runs but includes intermediate exp.data nodes and caused the ExpTest to fail - SQLFragment datas = new SQLFragment() - .append("SELECT d.Container, d.LSID, d.ObjectId, pa.CpasType AS pa_cpas_type FROM exp.Data d\n") - .append("INNER JOIN exp.DataInput di ON d.rowId = di.dataId\n") - .append("INNER JOIN exp.ProtocolApplication pa ON di.TargetApplicationId = pa.RowId\n") - .append("WHERE pa.RunId = ").append(runId).append(" AND pa.CpasType IN ('").append(ExperimentRun.name()).append("','").append(ExperimentRunOutput.name()).append("')"); + this._runId = runId; + this._runObjectId = runObjectId; + this._runLsid = runLsid; + this._runContainer = runContainer; + } - Collection> fromDataLsids = new ArrayList<>(); - Collection> toDataLsids = new ArrayList<>(); - new SqlSelector(getSchema(), datas).forEachMap(row -> { - if (ExperimentRun.name().equals(row.get("pa_cpas_type"))) - fromDataLsids.add(row); - else - toDataLsids.add(row); - }); - if (LOG.isDebugEnabled()) - { - if (!fromDataLsids.isEmpty()) - LOG.debug(" fromDataLsids:\n " + StringUtils.join(fromDataLsids, "\n ")); - if (!toDataLsids.isEmpty()) - LOG.debug(" toDataLsids:\n " + StringUtils.join(toDataLsids, "\n ")); - } - - SQLFragment materials = new SQLFragment() - .append("SELECT m.Container, m.LSID, m.CpasType, m.ObjectId, pa.CpasType AS pa_cpas_type FROM exp.material m\n") - .append("INNER JOIN exp.MaterialInput mi ON m.rowId = mi.materialId\n") - .append("INNER JOIN exp.ProtocolApplication pa ON mi.TargetApplicationId = pa.RowId\n") - .append("WHERE pa.RunId = ").append(runId).append(" AND pa.CpasType IN ('").append(ExperimentRun.name()).append("','").append(ExperimentRunOutput.name()).append("')"); - - Set fromCpasTypes = new HashSet<>(); - Collection> fromMaterialLsids = new ArrayList<>(); - Collection> toMaterialLsids = new ArrayList<>(); - new SqlSelector(getSchema(), materials).forEachMap(row -> { - String cpasType = (String)row.get("pa_cpas_type"); - if (ExperimentRun.name().equals(cpasType)) - { - fromCpasTypes.add(cpasType); - fromMaterialLsids.add(row); - } - else - { - toMaterialLsids.add(row); - } - }); + SyncRunEdges deleteFirst(boolean d) + { + deleteFirst = d; + return this; + } - Set> provenanceStartingInputs = emptySet(); - Set> provenanceFinalOutputs = emptySet(); + SyncRunEdges verifyEdgesNoInsert(boolean v) + { + verifyEdgesNoInsert = v; + return this; + } - ProvenanceService pvs = ProvenanceService.get(); - ProtocolApplication startProtocolApp = getStartingProtocolApplication(runId); - if (null != startProtocolApp) - { - provenanceStartingInputs = pvs.getProvenanceObjectIds(startProtocolApp.getRowId()); - } + SyncRunEdges invalidateClosureCache(boolean i) + { + invalidateClosureCache = i; + return this; + } - ProtocolApplication finalProtocolApp = getFinalProtocolApplication(runId); - if (null != finalProtocolApp) - { - provenanceFinalOutputs = pvs.getProvenanceObjectIds(finalProtocolApp.getRowId()); - } + void sync(@Nullable Map cpasTypeToObjectId) + { + syncRunEdges(_runId, _runObjectId, _runLsid, _runContainer, deleteFirst, verifyEdgesNoInsert, invalidateClosureCache, cpasTypeToObjectId); + } - // delete all existing edges for this run - if (deleteFirst) - removeEdgesForRun(runId); + private void syncRunEdges(int runId, Integer runObjectId, String runLsid, Container runContainer, boolean deleteFirst, boolean verifyEdgesNoInsert, boolean invalidateClosureCache, @Nullable Map cpasTypeToObjectId) + { + // don't do any updates if we are just verifying + if (verifyEdgesNoInsert) + deleteFirst = false; - int edgeCount = fromDataLsids.size() + fromMaterialLsids.size() + toDataLsids.size() + toMaterialLsids.size() + provenanceStartingInputs.size() + provenanceFinalOutputs.size(); - LOG.debug(String.format(" edge counts: input data=%d, input materials=%d, output data=%d, output materials=%d, input prov=%d, output prov=%d, total=%d", - fromDataLsids.size(), fromMaterialLsids.size(), toDataLsids.size(), toMaterialLsids.size(), provenanceStartingInputs.size(), provenanceFinalOutputs.size(), edgeCount)); + CPUTimer timer = new CPUTimer("sync edges"); + timer.start(); - if (edgeCount > 0) + LOG.debug((verifyEdgesNoInsert ? "Verifying" : "Rebuilding") + " edges for runId " + runId); + try (DbScope.Transaction tx = getExpSchema().getScope().ensureTransaction()) { - // ensure the run has an exp.object - if (null == runObjectId || 0 == runObjectId) + // NOTE: Originally, we just filtered exp.data by runId. This works for most runs but includes intermediate exp.data nodes and caused the ExpTest to fail + SQLFragment datas = new SQLFragment() + .append("SELECT d.Container, d.LSID, d.ObjectId, pa.CpasType AS pa_cpas_type FROM exp.Data d\n") + .append("INNER JOIN exp.DataInput di ON d.rowId = di.dataId\n") + .append("INNER JOIN exp.ProtocolApplication pa ON di.TargetApplicationId = pa.RowId\n") + .append("WHERE pa.RunId = ").append(runId).append(" AND pa.CpasType IN ('").append(ExperimentRun.name()).append("','").append(ExperimentRunOutput.name()).append("')"); + + Collection> fromDataLsids = new ArrayList<>(); + Collection> toDataLsids = new ArrayList<>(); + new SqlSelector(getSchema(), datas).forEachMap(row -> { + if (ExperimentRun.name().equals(row.get("pa_cpas_type"))) + fromDataLsids.add(row); + else + toDataLsids.add(row); + }); + if (LOG.isDebugEnabled()) { - if (LOG.isDebugEnabled()) - { - OntologyObject runObj = OntologyManager.getOntologyObject(runContainer, runLsid); - if (runObj == null) - LOG.debug(" run exp.object is null, creating: " + runLsid); - } - if (!verifyEdgesNoInsert) - runObjectId = OntologyManager.ensureObject(runContainer, runLsid, (Integer) null); + if (!fromDataLsids.isEmpty()) + LOG.debug(" fromDataLsids:\n " + StringUtils.join(fromDataLsids, "\n ")); + if (!toDataLsids.isEmpty()) + LOG.debug(" toDataLsids:\n " + StringUtils.join(toDataLsids, "\n ")); } - Map> allDatasByLsid = new HashMap<>(); - fromDataLsids.forEach(row -> allDatasByLsid.put((String) row.get("lsid"), row)); - toDataLsids.forEach(row -> allDatasByLsid.put((String) row.get("lsid"), row)); - if (!verifyEdgesNoInsert) - ensureNodeObjects(getTinfoData(), allDatasByLsid, cpasTypeToObjectId != null ? cpasTypeToObjectId : new HashMap<>()); - - Map> allMaterialsByLsid = new HashMap<>(); - fromMaterialLsids.forEach(row -> allMaterialsByLsid.put((String) row.get("lsid"), row)); - toMaterialLsids.forEach(row -> allMaterialsByLsid.put((String) row.get("lsid"), row)); - if (!verifyEdgesNoInsert) - ensureNodeObjects(getTinfoMaterial(), allMaterialsByLsid, cpasTypeToObjectId != null ? cpasTypeToObjectId : new HashMap<>()); - - List> params = new ArrayList<>(edgeCount); + SQLFragment materials = new SQLFragment() + .append("SELECT m.Container, m.LSID, m.CpasType, m.ObjectId, pa.CpasType AS pa_cpas_type FROM exp.material m\n") + .append("INNER JOIN exp.MaterialInput mi ON m.rowId = mi.materialId\n") + .append("INNER JOIN exp.ProtocolApplication pa ON mi.TargetApplicationId = pa.RowId\n") + .append("WHERE pa.RunId = ").append(runId).append(" AND pa.CpasType IN ('").append(ExperimentRun.name()).append("','").append(ExperimentRunOutput.name()).append("')"); + + Set toCpasTypes = new HashSet<>(); + Collection> fromMaterialLsids = new ArrayList<>(); + Collection> toMaterialLsids = new ArrayList<>(); + new SqlSelector(getSchema(), materials).forEachMap(row -> { + if (ExperimentRun.name().equals(row.get("pa_cpas_type"))) + { + fromMaterialLsids.add(row); + } + else + { + toMaterialLsids.add(row); + toCpasTypes.add((String)row.get("cpastype")); + } + }); - // - // from lsid -> run lsid - // + Set> provenanceStartingInputs = emptySet(); + Set> provenanceFinalOutputs = emptySet(); - Set seen = new HashSet<>(); - for (Map fromDataLsid : fromDataLsids) + ProvenanceService pvs = ProvenanceService.get(); + ProtocolApplication startProtocolApp = getStartingProtocolApplication(runId); + if (null != startProtocolApp) { - assert null != fromDataLsid.get("objectid"); - int objectid = (Integer)fromDataLsid.get("objectid"); - if (seen.add(objectid)) - prepEdgeForInsert(params, objectid, runObjectId, runId); + provenanceStartingInputs = pvs.getProvenanceObjectIds(startProtocolApp.getRowId()); } - for (Map fromMaterialLsid : fromMaterialLsids) + ProtocolApplication finalProtocolApp = getFinalProtocolApplication(runId); + if (null != finalProtocolApp) { - assert null != fromMaterialLsid.get("objectid"); - int objectid = (Integer)fromMaterialLsid.get("objectid"); - if (seen.add(objectid)) - prepEdgeForInsert(params, objectid, runObjectId, runId); + provenanceFinalOutputs = pvs.getProvenanceObjectIds(finalProtocolApp.getRowId()); } - if (!provenanceStartingInputs.isEmpty()) + // delete all existing edges for this run + if (deleteFirst) + removeEdgesForRun(runId); + + int edgeCount = fromDataLsids.size() + fromMaterialLsids.size() + toDataLsids.size() + toMaterialLsids.size() + provenanceStartingInputs.size() + provenanceFinalOutputs.size(); + LOG.debug(String.format(" edge counts: input data=%d, input materials=%d, output data=%d, output materials=%d, input prov=%d, output prov=%d, total=%d", + fromDataLsids.size(), fromMaterialLsids.size(), toDataLsids.size(), toMaterialLsids.size(), provenanceStartingInputs.size(), provenanceFinalOutputs.size(), edgeCount)); + + if (edgeCount > 0) { - for (Pair pair : provenanceStartingInputs) + // ensure the run has an exp.object + if (null == runObjectId || 0 == runObjectId) { - Integer fromId = pair.first; - if (null != fromId) + if (LOG.isDebugEnabled()) { - if (seen.add(fromId)) - prepEdgeForInsert(params, fromId, runObjectId, runId); + OntologyObject runObj = OntologyManager.getOntologyObject(runContainer, runLsid); + if (runObj == null) + LOG.debug(" run exp.object is null, creating: " + runLsid); } + if (!verifyEdgesNoInsert) + runObjectId = OntologyManager.ensureObject(runContainer, runLsid, (Integer) null); } - } - // - // run lsid -> to lsid - // + Map> allDatasByLsid = new HashMap<>(); + fromDataLsids.forEach(row -> allDatasByLsid.put((String) row.get("lsid"), row)); + toDataLsids.forEach(row -> allDatasByLsid.put((String) row.get("lsid"), row)); + if (!verifyEdgesNoInsert) + ensureNodeObjects(getTinfoData(), allDatasByLsid, cpasTypeToObjectId != null ? cpasTypeToObjectId : new HashMap<>()); - seen = new HashSet<>(); - for (Map toDataLsid : toDataLsids) - { - int objectid = (Integer)toDataLsid.get("objectid"); - if (seen.add(objectid)) - prepEdgeForInsert(params, runObjectId, objectid, runId); - } + Map> allMaterialsByLsid = new HashMap<>(); + fromMaterialLsids.forEach(row -> allMaterialsByLsid.put((String) row.get("lsid"), row)); + toMaterialLsids.forEach(row -> allMaterialsByLsid.put((String) row.get("lsid"), row)); + if (!verifyEdgesNoInsert) + ensureNodeObjects(getTinfoMaterial(), allMaterialsByLsid, cpasTypeToObjectId != null ? cpasTypeToObjectId : new HashMap<>()); - for (Map toMaterialLsid : toMaterialLsids) - { - int objectid = (Integer)toMaterialLsid.get("objectid"); - if (seen.add(objectid)) - prepEdgeForInsert(params, runObjectId, objectid, runId); - } + List> params = new ArrayList<>(edgeCount); - if (!provenanceFinalOutputs.isEmpty()) - { - for (Pair pair : provenanceFinalOutputs) + // + // from lsid -> run lsid + // + + Set seen = new HashSet<>(); + for (Map fromDataLsid : fromDataLsids) + { + assert null != fromDataLsid.get("objectid"); + int objectid = (Integer)fromDataLsid.get("objectid"); + if (seen.add(objectid)) + prepEdgeForInsert(params, objectid, runObjectId, runId); + } + + for (Map fromMaterialLsid : fromMaterialLsids) { - Integer toObjectId = pair.second; - if (null != toObjectId) + assert null != fromMaterialLsid.get("objectid"); + int objectid = (Integer)fromMaterialLsid.get("objectid"); + if (seen.add(objectid)) + prepEdgeForInsert(params, objectid, runObjectId, runId); + } + + if (!provenanceStartingInputs.isEmpty()) + { + for (Pair pair : provenanceStartingInputs) { - if (seen.add(toObjectId)) - prepEdgeForInsert(params, runObjectId, toObjectId, runId); + Integer fromId = pair.first; + if (null != fromId) + { + if (seen.add(fromId)) + prepEdgeForInsert(params, fromId, runObjectId, runId); + } } } - } - if (verifyEdgesNoInsert) - verifyEdges(runId, runObjectId, params); + // + // run lsid -> to lsid + // + + seen = new HashSet<>(); + for (Map toDataLsid : toDataLsids) + { + int objectid = (Integer)toDataLsid.get("objectid"); + if (seen.add(objectid)) + prepEdgeForInsert(params, runObjectId, objectid, runId); + } + + for (Map toMaterialLsid : toMaterialLsids) + { + int objectid = (Integer)toMaterialLsid.get("objectid"); + if (seen.add(objectid)) + prepEdgeForInsert(params, runObjectId, objectid, runId); + } + + if (!provenanceFinalOutputs.isEmpty()) + { + for (Pair pair : provenanceFinalOutputs) + { + Integer toObjectId = pair.second; + if (null != toObjectId) + { + if (seen.add(toObjectId)) + prepEdgeForInsert(params, runObjectId, toObjectId, runId); + } + } + } + + if (verifyEdgesNoInsert) + verifyEdges(runId, runObjectId, params); + else + { + insertEdges(params); + + } + } else { - insertEdges(params); - + if (verifyEdgesNoInsert) + verifyEdges(runId, runObjectId, Collections.emptyList()); } - } - else - { - if (verifyEdgesNoInsert) - verifyEdges(runId, runObjectId, Collections.emptyList()); - } - tx.commit(); - timer.stop(); - LOG.debug(" " + (verifyEdgesNoInsert ? "verified" : "synced") + " edges in " + timer.getDuration()); + tx.commit(); + timer.stop(); + LOG.debug(" " + (verifyEdgesNoInsert ? "verified" : "synced") + " edges in " + timer.getDuration()); - if (!verifyEdgesNoInsert && !fromCpasTypes.isEmpty()) - { - fromCpasTypes.forEach((type) -> ClosureQueryHelper.invalidateCpasType(type)); + if (!verifyEdgesNoInsert && invalidateClosureCache) + { + toCpasTypes.forEach(type -> ClosureQueryHelper.invalidateMaterialsForRun(type, runId)); + } } } } @@ -3206,7 +3259,11 @@ public void rebuildAllEdges() String runLsid = (String)run.get("lsid"); String containerId = (String)run.get("container"); Container runContainer = ContainerManager.getForId(containerId); - syncRunEdges(runId, runObjectId, runLsid, runContainer, false, false, cpasTypeToObjectId); + new SyncRunEdges(runId, runObjectId, runLsid, runContainer) + .deleteFirst(false) + .verifyEdgesNoInsert(false) + .invalidateClosureCache(false) // don't do incremental invalidation calls + .sync(cpasTypeToObjectId); } } @@ -3219,11 +3276,16 @@ public void rebuildAllEdges() ClosureQueryHelper.invalidateAll(); } + public void verifyRunEdges(ExpRun run) { - syncRunEdges(run.getRowId(), run.getObjectId(), run.getLSID(), run.getContainer(), false, true, null); + new SyncRunEdges(run) + .deleteFirst(false) + .verifyEdgesNoInsert(true) + .sync(null); } + public void verifyAllEdges(Container c, @Nullable Integer limit) { if (c.isRoot()) @@ -3264,7 +3326,10 @@ private void _verifyAllEdges(Container c, @Nullable Integer limit) String runLsid = (String)run.get("lsid"); String containerId = (String)run.get("container"); Container runContainer = ContainerManager.getForId(containerId); - syncRunEdges(runId, runObjectId, runLsid, runContainer, false, true, cpasTypeToObjectId); + new SyncRunEdges(runId, runObjectId, runLsid, runContainer) + .deleteFirst(false) + .verifyEdgesNoInsert(true) + .sync(cpasTypeToObjectId); runCount++; if (runCount % 1000 == 0) @@ -4609,7 +4674,6 @@ WHERE m3.rowId in (3592, 3593, 3594) .append(")"); return ExpRunImpl.fromRuns(getRunsForRunIds(sql)); - } private Collection getDerivedRunsFromMaterial(Collection materialIds) @@ -6476,7 +6540,10 @@ public void saveBatch() throws SQLException, XarFormatException, ValidationExcep Map cpasTypeToObjectId = new HashMap<>(); for (var er : runLsidToRowId.values()) { - syncRunEdges(er.getRowId(), er.getObjectId(), er.getLSID(), _container, false, false, cpasTypeToObjectId); + new SyncRunEdges(er.getExpObject()) + .deleteFirst(false) + .verifyEdgesNoInsert(false) + .sync(cpasTypeToObjectId); } } From 20f4ef20597b6c5ac621c0652f12b6c96f25373c Mon Sep 17 00:00:00 2001 From: Matthew Bellew Date: Tue, 15 Mar 2022 14:48:33 -0700 Subject: [PATCH 4/9] MERGE USING --- .../experiment/api/ClosureQueryHelper.java | 21 +++++++++++++++---- .../experiment/api/ExpMaterialTableImpl.java | 2 ++ 2 files changed, 19 insertions(+), 4 deletions(-) diff --git a/experiment/src/org/labkey/experiment/api/ClosureQueryHelper.java b/experiment/src/org/labkey/experiment/api/ClosureQueryHelper.java index a643f467996..ced61bc175e 100644 --- a/experiment/src/org/labkey/experiment/api/ClosureQueryHelper.java +++ b/experiment/src/org/labkey/experiment/api/ClosureQueryHelper.java @@ -60,7 +60,7 @@ WITH RECURSIVE CTE_ AS ( static String pgMaterialClosureSql = String.format(""" SELECT Start_, CASE WHEN COUNT(*) = 1 THEN MIN(rowId) ELSE -1 * COUNT(*) END AS rowId, targetId - /* INTO */ + /*INTO*/ FROM ( SELECT Start_, End_, COALESCE(material.rowid, dataclass.rowid) as rowId, @@ -204,9 +204,22 @@ private static void incrementalRecompute(String sourceLSID, SQLFragment from) SQLFragment selectInto = selectIntoSql(getScope().getSqlDialect(), from, tempTableName); new SqlExecutor(getScope()).execute(selectInto); - SQLFragment upsert = new SQLFragment() - .append("INSERT INTO temp.${NAME} (Start_, rowId, targetid)\n").append("SELECT Start, RowId, targetId FROM temp.").append(tempTableName).append(" TMP\n") - .append("ON CONFLICT(Start_,targetId) UPDATE SET rowId = EXCLUDED.rowId"); + SQLFragment upsert; + if (getScope().getSqlDialect().isPostgreSQL()) + { + upsert = new SQLFragment() + .append("INSERT INTO temp.${NAME} (Start_, rowId, targetid)\n") + .append("SELECT Start_, RowId, targetId FROM temp.").append(tempTableName).append(" TMP\n") + .append("ON CONFLICT(Start_,targetId) DO UPDATE SET rowId = EXCLUDED.rowId;"); + } + else + { + upsert = new SQLFragment() + .append("MERGE temp.${NAME} AS Target\n") + .append("USING (SELECT Start_, RowId, targetId FROM temp.").append(tempTableName).append(") AS Source ON Target.Start_=Source.Start_ AND Target.targetid=Source.targetId\n") + .append("WHEN MATCHED THEN UPDATE SET Target.targetId = Source.targetId\n") + .append("WHEN NOT MATCHED THEN INSERT (Start_, rowId, targetid) VALUES (Source.Start_, Source.rowId, Source.targetId);"); + } helper.upsert(upsert); } diff --git a/experiment/src/org/labkey/experiment/api/ExpMaterialTableImpl.java b/experiment/src/org/labkey/experiment/api/ExpMaterialTableImpl.java index 9c64ad4339a..b979b31047c 100644 --- a/experiment/src/org/labkey/experiment/api/ExpMaterialTableImpl.java +++ b/experiment/src/org/labkey/experiment/api/ExpMaterialTableImpl.java @@ -696,6 +696,8 @@ protected ContainerFilter getLookupContainerFilter() @Override public @Nullable ColumnInfo createLookupColumn(ColumnInfo parent, String displayField) { + if (null == displayField) + return null; var target = SampleTypeServiceImpl.get().getSampleTypeByType(displayField, _userSchema.getContainer()); if (null == target) target = SampleTypeServiceImpl.get().getSampleType(_userSchema.getContainer(), _userSchema.getUser(), displayField); From 57894f0196bb7e6cc9fe3c5d1497e949dc252495 Mon Sep 17 00:00:00 2001 From: Matthew Bellew Date: Tue, 15 Mar 2022 17:09:18 -0700 Subject: [PATCH 5/9] lots of lookups --- .../experiment/api/ClosureQueryHelper.java | 33 +++-- .../experiment/api/ExpMaterialTableImpl.java | 133 ++++++++++++++---- 2 files changed, 124 insertions(+), 42 deletions(-) diff --git a/experiment/src/org/labkey/experiment/api/ClosureQueryHelper.java b/experiment/src/org/labkey/experiment/api/ClosureQueryHelper.java index ced61bc175e..fe65c904778 100644 --- a/experiment/src/org/labkey/experiment/api/ClosureQueryHelper.java +++ b/experiment/src/org/labkey/experiment/api/ClosureQueryHelper.java @@ -15,6 +15,7 @@ import org.labkey.api.data.TempTableTracker; import org.labkey.api.data.dialect.SqlDialect; import org.labkey.api.exp.api.ExpDataClass; +import org.labkey.api.exp.api.ExpObject; import org.labkey.api.exp.api.ExpSampleType; import org.labkey.api.query.ExprColumn; import org.labkey.api.query.QueryForeignKey; @@ -58,7 +59,7 @@ WITH RECURSIVE CTE_ AS ( ) """, MAX_LINEAGE_LOOKUP_DEPTH); - static String pgMaterialClosureSql = String.format(""" + static String pgMaterialClosureSql = """ SELECT Start_, CASE WHEN COUNT(*) = 1 THEN MIN(rowId) ELSE -1 * COUNT(*) END AS rowId, targetId /*INTO*/ FROM ( @@ -70,7 +71,7 @@ SELECT Start_, CASE WHEN COUNT(*) = 1 THEN MIN(rowId) ELSE -1 * COUNT(*) END AS LEFT OUTER JOIN exp.data on End_ = data.objectId LEFT OUTER JOIN exp.dataclass ON data.cpasType = dataclass.lsid WHERE Depth_ > 0 AND materialsource.rowid IS NOT NULL OR dataclass.rowid IS NOT NULL) _inner_ GROUP BY targetId, Start_ - """, MAX_LINEAGE_LOOKUP_DEPTH); + """; static String mssqlMaterialClosureCTE = String.format(""" @@ -142,16 +143,24 @@ static SQLFragment selectSql(SqlDialect d, SQLFragment from) * This can be used to add a column directly to a exp table, or to create a column * in an intermediate fake lookup table */ - static MutableColumnInfo createLineageLookupColumn(final ColumnInfo fkRowId, ExpSampleType source, ExpSampleType target) + static MutableColumnInfo createLineageLookupColumn(final ColumnInfo fkRowId, ExpObject source, ExpObject target) { + if (!(source instanceof ExpSampleType) && !(source instanceof ExpDataClass)) + throw new IllegalStateException(); + if (!(target instanceof ExpSampleType) && !(target instanceof ExpDataClass)) + throw new IllegalStateException(); + TableInfo parentTable = fkRowId.getParentTable(); - var ret = new ExprColumn(parentTable, target.getLSID(), new SQLFragment("#ERROR#"), JdbcType.INTEGER) + var ret = new ExprColumn(parentTable, target.getName(), new SQLFragment("#ERROR#"), JdbcType.INTEGER) { @Override public SQLFragment getValueSql(String tableAlias) { SQLFragment objectId = fkRowId.getValueSql(tableAlias); - return ClosureQueryHelper.getValueSql(source.getLSID(), objectId, target); + String sourceLsid = source instanceof ExpSampleType ss ? ss.getLSID() : source instanceof ExpDataClass dc ? dc.getLSID() : null; + if (sourceLsid == null) + return new SQLFragment(" NULL "); + return ClosureQueryHelper.getValueSql(sourceLsid, objectId, target); } }; ret.setLabel(target.getName()); @@ -161,15 +170,13 @@ public SQLFragment getValueSql(String tableAlias) } - public static SQLFragment getValueSql(String sourceLSID, SQLFragment objectId, ExpSampleType target) - { - return getValueSql(sourceLSID, objectId, "m" + target.getRowId()); - } - - - public static SQLFragment getValueSql(String sourceLSID, SQLFragment objectId, ExpDataClass target) + public static SQLFragment getValueSql(String sourceLSID, SQLFragment objectId, ExpObject target) { - return getValueSql(sourceLSID, objectId, "d" + target.getRowId()); + if (target instanceof ExpSampleType st) + return getValueSql(sourceLSID, objectId, "m" + st.getRowId()); + if (target instanceof ExpDataClass dc) + return getValueSql(sourceLSID, objectId, "d" + dc.getRowId()); + throw new IllegalStateException(); } diff --git a/experiment/src/org/labkey/experiment/api/ExpMaterialTableImpl.java b/experiment/src/org/labkey/experiment/api/ExpMaterialTableImpl.java index b979b31047c..1fa746c2a4c 100644 --- a/experiment/src/org/labkey/experiment/api/ExpMaterialTableImpl.java +++ b/experiment/src/org/labkey/experiment/api/ExpMaterialTableImpl.java @@ -32,6 +32,7 @@ import org.labkey.api.exp.OntologyManager; import org.labkey.api.exp.PropertyColumn; import org.labkey.api.exp.api.ExpMaterial; +import org.labkey.api.exp.api.ExpObject; import org.labkey.api.exp.api.ExpProtocol; import org.labkey.api.exp.api.ExpSampleType; import org.labkey.api.exp.api.ExperimentService; @@ -62,6 +63,7 @@ import org.labkey.api.query.UserSchema; import org.labkey.api.query.column.BuiltInColumnTypes; import org.labkey.api.search.SearchService; +import org.labkey.api.security.User; import org.labkey.api.security.UserPrincipal; import org.labkey.api.security.permissions.DeletePermission; import org.labkey.api.security.permissions.InsertPermission; @@ -82,6 +84,7 @@ import java.io.UncheckedIOException; import java.util.ArrayList; import java.util.Arrays; +import java.util.Collection; import java.util.Collections; import java.util.HashMap; import java.util.HashSet; @@ -671,45 +674,117 @@ protected ContainerFilter getLookupContainerFilter() setDefaultVisibleColumns(defaultCols); - /* ADD SAMPLETYPE LINEAGE LOOKUP COLUMNS - * - * TODO - * which SAMPLETYPE tables do we add? - * do we want an intermediate fake lookup column? - * do we want to enforce uniqueness of lookup before creating lookup column? how? - * - * Probably best to add intermediate lookup column, to avoid TableSelector(ALL_COLUMNS) doing all this work. - * See SampleTypeUpdateServiceDI.getMaterialMap(). - */ if (null != _ss) { - var sts = SampleTypeServiceImpl.get().getSampleTypes(_userSchema.getContainer(), _userSchema.getUser(), false); - if (null != sts && !sts.isEmpty()) + MutableColumnInfo wrappedRowId = wrapColumn("LineageLookupTypes", _rootTable.getColumn("rowid")); + wrappedRowId.setIsUnselectable(true); + wrappedRowId.setReadOnly(true); + wrappedRowId.setCalculated(true); + wrappedRowId.setRequired(false); + wrappedRowId.setFk(new AbstractForeignKey(getUserSchema(),null) { - MutableColumnInfo wrappedRowId = wrapColumn("LineageLookupPlaceHolder", _rootTable.getColumn("rowid")); - wrappedRowId.setIsUnselectable(true); - wrappedRowId.setReadOnly(true); - wrappedRowId.setCalculated(true); - wrappedRowId.setRequired(false); - wrappedRowId.setFk(new AbstractForeignKey(getUserSchema(),null) + @Override + public TableInfo getLookupTableInfo() + { + return new LineageLookupTypesTableInfo(); + } + + @Override + public @Nullable ColumnInfo createLookupColumn(ColumnInfo parent, String displayField) + { + ColumnInfo lk = getLookupTableInfo().getColumn(displayField); + if (null == lk) + return null; + var ret = new ExprColumn(parent.getParentTable(), new FieldKey(parent.getFieldKey(),lk.getName()), null, JdbcType.INTEGER) + { + @Override + public SQLFragment getValueSql(String tableAlias) + { + return parent.getValueSql(tableAlias); + } + }; + ret.setFk(lk.getFk()); + return ret; + } + + @Override + public StringExpression getURL(ColumnInfo parent) + { + return null; + } + }); + addColumn(wrappedRowId); + } + } + + enum LookupType + { + SampleType("Materials") + { + @Override + Collection getInstances(Container c, User u) + { + return SampleTypeServiceImpl.get().getSampleTypes(c, u,false); + } + @Override + ExpObject getInstance(Container c, User u, String name) + { + return SampleTypeServiceImpl.get().getSampleType(c, u, name); + } + }, + DataClass("Data") + { + @Override + Collection getInstances(Container c, User u) + { + return ExperimentServiceImpl.get().getDataClasses(c, u,false); + } + @Override + ExpObject getInstance(Container c, User u, String name) + { + return ExperimentServiceImpl.get().getDataClass(c, u, name); + } + }; + + final String lookupName; + + LookupType(String lookupName) + { + this.lookupName = lookupName; + } + + abstract Collection getInstances(Container c, User u); + abstract ExpObject getInstance(Container c, User u, String name); + }; + + + private class LineageLookupTypesTableInfo extends VirtualTable + { + LineageLookupTypesTableInfo() + { + super(ExpMaterialTableImpl.this.getSchema(), "LineageLookupTypes", ExpMaterialTableImpl.this.getUserSchema()); + + for (var lk : LookupType.values()) + { + var col = new BaseColumnInfo(lk.lookupName, this, JdbcType.INTEGER); + col.setIsUnselectable(true); + col.setFk(new AbstractForeignKey(getUserSchema(),null) { @Override public @Nullable ColumnInfo createLookupColumn(ColumnInfo parent, String displayField) { if (null == displayField) return null; - var target = SampleTypeServiceImpl.get().getSampleTypeByType(displayField, _userSchema.getContainer()); - if (null == target) - target = SampleTypeServiceImpl.get().getSampleType(_userSchema.getContainer(), _userSchema.getUser(), displayField); + var target = lk.getInstance(_userSchema.getContainer(), _userSchema.getUser(), displayField); if (null == target) return null; return ClosureQueryHelper.createLineageLookupColumn(parent, _ss, target); } @Override - public @Nullable TableInfo getLookupTableInfo() + public TableInfo getLookupTableInfo() { - return new PlaceHolderLineageLookupTableInfo(); + return new LineageLookupTableInfo(lk); } @Override @@ -718,20 +793,19 @@ public StringExpression getURL(ColumnInfo parent) return null; } }); - addColumn(wrappedRowId); + addColumn(col); } } } - private class PlaceHolderLineageLookupTableInfo extends VirtualTable + private class LineageLookupTableInfo extends VirtualTable { - PlaceHolderLineageLookupTableInfo() + LineageLookupTableInfo(LookupType type) { - super(ExpMaterialTableImpl.this.getSchema(), "LineageLookupPlaceHolder", ExpMaterialTableImpl.this.getUserSchema()); - List sts = SampleTypeServiceImpl.get().getSampleTypes(_userSchema.getContainer(), _userSchema.getUser(), false); + super(ExpMaterialTableImpl.this.getSchema(), "Lineage Lookup", ExpMaterialTableImpl.this.getUserSchema()); ColumnInfo wrap = new BaseColumnInfo("rowid", this, JdbcType.INTEGER); - for (ExpSampleTypeImpl target : sts) + for (var target : type.getInstances(_userSchema.getContainer(), _userSchema.getUser())) addColumn(ClosureQueryHelper.createLineageLookupColumn(wrap, _ss, target)); } @@ -1022,6 +1096,7 @@ public boolean hasPermission(@NotNull UserPrincipal user, @NotNull Class>> getUniqueIndices() From db12581bb0c713d01cb3fce17150941cbd74a71e Mon Sep 17 00:00:00 2001 From: Matthew Bellew Date: Wed, 16 Mar 2022 12:52:25 -0700 Subject: [PATCH 6/9] move lookup code from ExpMaterialTableImpl to ClosureQueryHelper --- .../experiment/api/ClosureQueryHelper.java | 265 ++++++++++++++++-- .../experiment/api/ExpMaterialTableImpl.java | 144 +--------- 2 files changed, 246 insertions(+), 163 deletions(-) diff --git a/experiment/src/org/labkey/experiment/api/ClosureQueryHelper.java b/experiment/src/org/labkey/experiment/api/ClosureQueryHelper.java index fe65c904778..13cb438f250 100644 --- a/experiment/src/org/labkey/experiment/api/ClosureQueryHelper.java +++ b/experiment/src/org/labkey/experiment/api/ClosureQueryHelper.java @@ -1,8 +1,12 @@ package org.labkey.experiment.api; import org.apache.commons.lang3.StringUtils; +import org.jetbrains.annotations.NotNull; import org.jetbrains.annotations.Nullable; +import org.labkey.api.data.AbstractForeignKey; +import org.labkey.api.data.BaseColumnInfo; import org.labkey.api.data.ColumnInfo; +import org.labkey.api.data.Container; import org.labkey.api.data.CoreSchema; import org.labkey.api.data.DbSchema; import org.labkey.api.data.DbScope; @@ -13,16 +17,24 @@ import org.labkey.api.data.SqlExecutor; import org.labkey.api.data.TableInfo; import org.labkey.api.data.TempTableTracker; +import org.labkey.api.data.VirtualTable; import org.labkey.api.data.dialect.SqlDialect; import org.labkey.api.exp.api.ExpDataClass; import org.labkey.api.exp.api.ExpObject; import org.labkey.api.exp.api.ExpSampleType; import org.labkey.api.query.ExprColumn; +import org.labkey.api.query.FieldKey; +import org.labkey.api.query.FilteredTable; import org.labkey.api.query.QueryForeignKey; import org.labkey.api.query.UserSchema; +import org.labkey.api.security.User; +import org.labkey.api.util.HeartBeat; +import org.labkey.api.util.StringExpression; +import java.util.Collection; import java.util.Collections; import java.util.HashMap; +import java.util.LinkedHashMap; import java.util.Map; import java.util.Objects; import java.util.concurrent.TimeUnit; @@ -33,10 +45,19 @@ public class ClosureQueryHelper { + final static long CACHE_INVALIDATION_INTERVAL = TimeUnit.MINUTES.toMillis(5); + final static long CACHE_LRU_AGE_OUT_INTERVAL = TimeUnit.MINUTES.toMillis(30); + + + /* TODO/CONSIDER every SampleType and Dataclass should have a unique ObjectId so it can be stored as an in lineage tables (e.g. edge/closure tables) */ - static final Map queryHelpers = Collections.synchronizedMap(new HashMap<>()); - static final Map invalidators = Collections.synchronizedMap(new HashMap<>()); + record ClosureTable(MaterializedQueryHelper helper, AtomicInteger counter, TableType type, String lsid) {}; + + static final Map queryHelpers = Collections.synchronizedMap(new HashMap<>()); + // use this as a separate LRU implementation, because I only want to track calls to getValueSql() not other calls to queryHelpers.get() + static final Map lruQueryHelpers = new LinkedHashMap(100,0.75f,true); + static final int MAX_LINEAGE_LOOKUP_DEPTH = 10; @@ -150,6 +171,8 @@ static MutableColumnInfo createLineageLookupColumn(final ColumnInfo fkRowId, Exp if (!(target instanceof ExpSampleType) && !(target instanceof ExpDataClass)) throw new IllegalStateException(); + final TableType sourceType = source instanceof ExpSampleType ss ? TableType.SampleType : TableType.DataClass; + TableInfo parentTable = fkRowId.getParentTable(); var ret = new ExprColumn(parentTable, target.getName(), new SQLFragment("#ERROR#"), JdbcType.INTEGER) { @@ -157,10 +180,10 @@ static MutableColumnInfo createLineageLookupColumn(final ColumnInfo fkRowId, Exp public SQLFragment getValueSql(String tableAlias) { SQLFragment objectId = fkRowId.getValueSql(tableAlias); - String sourceLsid = source instanceof ExpSampleType ss ? ss.getLSID() : source instanceof ExpDataClass dc ? dc.getLSID() : null; + String sourceLsid = source.getLSID(); if (sourceLsid == null) return new SQLFragment(" NULL "); - return ClosureQueryHelper.getValueSql(sourceLsid, objectId, target); + return ClosureQueryHelper.getValueSql(sourceType, sourceLsid, objectId, target); } }; ret.setLabel(target.getName()); @@ -170,19 +193,19 @@ public SQLFragment getValueSql(String tableAlias) } - public static SQLFragment getValueSql(String sourceLSID, SQLFragment objectId, ExpObject target) + public static SQLFragment getValueSql(TableType type, String sourceLSID, SQLFragment objectId, ExpObject target) { if (target instanceof ExpSampleType st) - return getValueSql(sourceLSID, objectId, "m" + st.getRowId()); + return getValueSql(type, sourceLSID, objectId, "m" + st.getRowId()); if (target instanceof ExpDataClass dc) - return getValueSql(sourceLSID, objectId, "d" + dc.getRowId()); + return getValueSql(type, sourceLSID, objectId, "d" + dc.getRowId()); throw new IllegalStateException(); } - private static SQLFragment getValueSql(String sourceLSID, SQLFragment objectId, String targetId) + private static SQLFragment getValueSql(TableType type, String sourceLSID, SQLFragment objectId, String targetId) { - MaterializedQueryHelper helper = getClosureHelper(sourceLSID, true); + MaterializedQueryHelper helper = getClosureHelper(type, sourceLSID, true); return new SQLFragment() .append("(SELECT rowId FROM ") @@ -198,7 +221,7 @@ private static SQLFragment getValueSql(String sourceLSID, SQLFragment objectId, private static void incrementalRecompute(String sourceLSID, SQLFragment from) { // if there's nothing cached, we don't need to do incremental - MaterializedQueryHelper helper = getClosureHelper(sourceLSID, false); + MaterializedQueryHelper helper = getClosureHelper(null, sourceLSID, false); if (null == helper || !helper.isCached(null)) return; @@ -232,7 +255,7 @@ private static void incrementalRecompute(String sourceLSID, SQLFragment from) } catch (Exception x) { - getInvalidationCounter(sourceLSID).incrementAndGet(); + invalidate(sourceLSID); throw x; } finally @@ -247,8 +270,11 @@ public static void invalidateAll() { synchronized (queryHelpers) { - for (var h : queryHelpers.values()) - h.uncache(null); + for (var c : queryHelpers.values()) + { + c.counter.incrementAndGet(); + c.helper.uncache(null); + } } } @@ -281,28 +307,71 @@ public static void invalidateMaterialsForRun(String sourceTypeLsid, int runId) } - private static MaterializedQueryHelper getClosureHelper(String sourceLSID, boolean computeIfAbsent) + private static MaterializedQueryHelper getClosureHelper(TableType type, String sourceLSID, boolean computeIfAbsent) { + ClosureTable closure; + if (!computeIfAbsent) - return queryHelpers.get(sourceLSID); + { + closure = queryHelpers.get(sourceLSID); + return null==closure ? null : closure.helper; + } - return queryHelpers.computeIfAbsent(sourceLSID, cpasType -> + if (null == type) + throw new IllegalStateException(); + + closure = queryHelpers.computeIfAbsent(sourceLSID, cpasType -> { SQLFragment from = new SQLFragment(" FROM exp.Material WHERE Material.cpasType = ? ").add(cpasType); SQLFragment selectInto = selectIntoSql(getScope().getSqlDialect(), from, null); - return new MaterializedQueryHelper.Builder("closure", DbSchema.getTemp().getScope(), selectInto) + + var helper = new MaterializedQueryHelper.Builder("closure", DbSchema.getTemp().getScope(), selectInto) .setIsSelectInto(true) .addIndex("CREATE UNIQUE INDEX uq_${NAME} ON temp.${NAME} (targetId,Start_)") - .maxTimeToCache(TimeUnit.MINUTES.toMillis(5)) - .addInvalidCheck(() -> String.valueOf(getInvalidationCounter(sourceLSID))) + .maxTimeToCache(CACHE_INVALIDATION_INTERVAL) + .addInvalidCheck(() -> getInvalidationCounterString(sourceLSID)) .build(); + return new ClosureTable(helper, new AtomicInteger(), type, sourceLSID); }); + + // update LRU + synchronized (lruQueryHelpers) + { + lruQueryHelpers.put(sourceLSID, HeartBeat.currentTimeMillis()); + checkStaleEntries(); + } + + return closure.helper; } - private static AtomicInteger getInvalidationCounter(String sourceLSID) + private static void checkStaleEntries() { - return invalidators.computeIfAbsent(sourceLSID, (key) -> new AtomicInteger()); + synchronized (lruQueryHelpers) + { + if (lruQueryHelpers.isEmpty()) + return; + var oldestEntry = lruQueryHelpers.entrySet().iterator().next(); + if (HeartBeat.currentTimeMillis() - oldestEntry.getValue() < CACHE_LRU_AGE_OUT_INTERVAL) + return; + queryHelpers.remove(oldestEntry.getKey()); + lruQueryHelpers.remove(oldestEntry.getKey()); + } + } + + + private static void invalidate(String sourceLSID) + { + var closure = queryHelpers.get(sourceLSID); + if (null != closure) + closure.counter.incrementAndGet(); + } + + + private static String getInvalidationCounterString(String sourceLSID) + { + var closure = queryHelpers.get(sourceLSID); + return null==closure ? "-1" : String.valueOf(closure.counter.get()); } @@ -310,4 +379,158 @@ private static DbScope getScope() { return CoreSchema.getInstance().getScope(); } + + + + + + + + /* + * Code to create the lineage parent lookup column and intermediate lookups that use ClosureQueryHelper + */ + + public static MutableColumnInfo createLineageLookupColumnInfo(String columnName, FilteredTable parent, ColumnInfo rowid, ExpObject source) + { + //MutableColumnInfo wrappedRowId = wrapColumn("LineageLookupTypes", _rootTable.getColumn("rowid")); + MutableColumnInfo wrappedRowId = parent.wrapColumn(columnName, rowid); + wrappedRowId.setIsUnselectable(true); + wrappedRowId.setReadOnly(true); + wrappedRowId.setCalculated(true); + wrappedRowId.setRequired(false); + wrappedRowId.setFk(new AbstractForeignKey(parent.getUserSchema(),parent.getContainerFilter()) + { + @Override + public TableInfo getLookupTableInfo() + { + return new LineageLookupTypesTableInfo(parent.getUserSchema(), source); + } + + @Override + public @Nullable ColumnInfo createLookupColumn(ColumnInfo parent, String displayField) + { + ColumnInfo lk = getLookupTableInfo().getColumn(displayField); + if (null == lk) + return null; + var ret = new ExprColumn(parent.getParentTable(), new FieldKey(parent.getFieldKey(),lk.getName()), null, JdbcType.INTEGER) + { + @Override + public SQLFragment getValueSql(String tableAlias) + { + return parent.getValueSql(tableAlias); + } + }; + ret.setFk(lk.getFk()); + return ret; + } + + @Override + public StringExpression getURL(ColumnInfo parent) + { + return null; + } + }); + return wrappedRowId; + } + + + enum TableType + { + SampleType("Materials") + { + @Override + Collection getInstances(Container c, User u) + { + return SampleTypeServiceImpl.get().getSampleTypes(c, u,false); + } + @Override + ExpObject getInstance(Container c, User u, String name) + { + return SampleTypeServiceImpl.get().getSampleType(c, u, name); + } + }, + DataClass("Data") + { + @Override + Collection getInstances(Container c, User u) + { + return ExperimentServiceImpl.get().getDataClasses(c, u,false); + } + @Override + ExpObject getInstance(Container c, User u, String name) + { + return ExperimentServiceImpl.get().getDataClass(c, u, name); + } + }; + + final String lookupName; + + TableType(String lookupName) + { + this.lookupName = lookupName; + } + + abstract Collection getInstances(Container c, User u); + abstract ExpObject getInstance(Container c, User u, String name); + }; + + + private static class LineageLookupTypesTableInfo extends VirtualTable + { + LineageLookupTypesTableInfo(UserSchema userSchema, ExpObject source) + { + super(userSchema.getDbSchema(), "LineageLookupTypes",userSchema); + + for (var lk : TableType.values()) + { + var col = new BaseColumnInfo(lk.lookupName, this, JdbcType.INTEGER); + col.setIsUnselectable(true); + col.setFk(new AbstractForeignKey(getUserSchema(),null) + { + @Override + public @Nullable ColumnInfo createLookupColumn(ColumnInfo parent, String displayField) + { + if (null == displayField) + return null; + var target = lk.getInstance(_userSchema.getContainer(), _userSchema.getUser(), displayField); + if (null == target) + return null; + return ClosureQueryHelper.createLineageLookupColumn(parent, source, target); + } + + @Override + public TableInfo getLookupTableInfo() + { + return new LineageLookupTableInfo(userSchema, source, lk); + } + + @Override + public StringExpression getURL(ColumnInfo parent) + { + return null; + } + }); + addColumn(col); + } + } + } + + + private static class LineageLookupTableInfo extends VirtualTable + { + LineageLookupTableInfo(UserSchema userSchema, ExpObject source, TableType type) + { + super(userSchema.getDbSchema(), "Lineage Lookup", userSchema); + ColumnInfo wrap = new BaseColumnInfo("rowid", this, JdbcType.INTEGER); + for (var target : type.getInstances(_userSchema.getContainer(), _userSchema.getUser())) + addColumn(ClosureQueryHelper.createLineageLookupColumn(wrap, source, target)); + } + + @Override + public @NotNull SQLFragment getFromSQL() + { + throw new IllegalStateException(); + } + } + } diff --git a/experiment/src/org/labkey/experiment/api/ExpMaterialTableImpl.java b/experiment/src/org/labkey/experiment/api/ExpMaterialTableImpl.java index 1fa746c2a4c..3a17da594e9 100644 --- a/experiment/src/org/labkey/experiment/api/ExpMaterialTableImpl.java +++ b/experiment/src/org/labkey/experiment/api/ExpMaterialTableImpl.java @@ -32,7 +32,6 @@ import org.labkey.api.exp.OntologyManager; import org.labkey.api.exp.PropertyColumn; import org.labkey.api.exp.api.ExpMaterial; -import org.labkey.api.exp.api.ExpObject; import org.labkey.api.exp.api.ExpProtocol; import org.labkey.api.exp.api.ExpSampleType; import org.labkey.api.exp.api.ExperimentService; @@ -63,7 +62,6 @@ import org.labkey.api.query.UserSchema; import org.labkey.api.query.column.BuiltInColumnTypes; import org.labkey.api.search.SearchService; -import org.labkey.api.security.User; import org.labkey.api.security.UserPrincipal; import org.labkey.api.security.permissions.DeletePermission; import org.labkey.api.security.permissions.InsertPermission; @@ -84,7 +82,6 @@ import java.io.UncheckedIOException; import java.util.ArrayList; import java.util.Arrays; -import java.util.Collection; import java.util.Collections; import java.util.HashMap; import java.util.HashSet; @@ -676,147 +673,11 @@ protected ContainerFilter getLookupContainerFilter() if (null != _ss) { - MutableColumnInfo wrappedRowId = wrapColumn("LineageLookupTypes", _rootTable.getColumn("rowid")); - wrappedRowId.setIsUnselectable(true); - wrappedRowId.setReadOnly(true); - wrappedRowId.setCalculated(true); - wrappedRowId.setRequired(false); - wrappedRowId.setFk(new AbstractForeignKey(getUserSchema(),null) - { - @Override - public TableInfo getLookupTableInfo() - { - return new LineageLookupTypesTableInfo(); - } - - @Override - public @Nullable ColumnInfo createLookupColumn(ColumnInfo parent, String displayField) - { - ColumnInfo lk = getLookupTableInfo().getColumn(displayField); - if (null == lk) - return null; - var ret = new ExprColumn(parent.getParentTable(), new FieldKey(parent.getFieldKey(),lk.getName()), null, JdbcType.INTEGER) - { - @Override - public SQLFragment getValueSql(String tableAlias) - { - return parent.getValueSql(tableAlias); - } - }; - ret.setFk(lk.getFk()); - return ret; - } - - @Override - public StringExpression getURL(ColumnInfo parent) - { - return null; - } - }); - addColumn(wrappedRowId); - } - } - - enum LookupType - { - SampleType("Materials") - { - @Override - Collection getInstances(Container c, User u) - { - return SampleTypeServiceImpl.get().getSampleTypes(c, u,false); - } - @Override - ExpObject getInstance(Container c, User u, String name) - { - return SampleTypeServiceImpl.get().getSampleType(c, u, name); - } - }, - DataClass("Data") - { - @Override - Collection getInstances(Container c, User u) - { - return ExperimentServiceImpl.get().getDataClasses(c, u,false); - } - @Override - ExpObject getInstance(Container c, User u, String name) - { - return ExperimentServiceImpl.get().getDataClass(c, u, name); - } - }; - - final String lookupName; - - LookupType(String lookupName) - { - this.lookupName = lookupName; - } - - abstract Collection getInstances(Container c, User u); - abstract ExpObject getInstance(Container c, User u, String name); - }; - - - private class LineageLookupTypesTableInfo extends VirtualTable - { - LineageLookupTypesTableInfo() - { - super(ExpMaterialTableImpl.this.getSchema(), "LineageLookupTypes", ExpMaterialTableImpl.this.getUserSchema()); - - for (var lk : LookupType.values()) - { - var col = new BaseColumnInfo(lk.lookupName, this, JdbcType.INTEGER); - col.setIsUnselectable(true); - col.setFk(new AbstractForeignKey(getUserSchema(),null) - { - @Override - public @Nullable ColumnInfo createLookupColumn(ColumnInfo parent, String displayField) - { - if (null == displayField) - return null; - var target = lk.getInstance(_userSchema.getContainer(), _userSchema.getUser(), displayField); - if (null == target) - return null; - return ClosureQueryHelper.createLineageLookupColumn(parent, _ss, target); - } - - @Override - public TableInfo getLookupTableInfo() - { - return new LineageLookupTableInfo(lk); - } - - @Override - public StringExpression getURL(ColumnInfo parent) - { - return null; - } - }); - addColumn(col); - } + MutableColumnInfo lineageLookup = ClosureQueryHelper.createLineageLookupColumnInfo("Parent Lookups", this, _rootTable.getColumn("rowid"), _ss); + addColumn(lineageLookup); } } - - private class LineageLookupTableInfo extends VirtualTable - { - LineageLookupTableInfo(LookupType type) - { - super(ExpMaterialTableImpl.this.getSchema(), "Lineage Lookup", ExpMaterialTableImpl.this.getUserSchema()); - ColumnInfo wrap = new BaseColumnInfo("rowid", this, JdbcType.INTEGER); - for (var target : type.getInstances(_userSchema.getContainer(), _userSchema.getUser())) - addColumn(ClosureQueryHelper.createLineageLookupColumn(wrap, _ss, target)); - } - - @Override - public @NotNull SQLFragment getFromSQL() - { - throw new IllegalStateException(); - } - } - - @Override public Domain getDomain() { @@ -1096,7 +957,6 @@ public boolean hasPermission(@NotNull UserPrincipal user, @NotNull Class>> getUniqueIndices() From 0c13fee51a93f409cd7e673157c159d37c12dfda Mon Sep 17 00:00:00 2001 From: Matthew Bellew Date: Wed, 16 Mar 2022 17:57:44 -0700 Subject: [PATCH 7/9] experimental feature (for now) --- .../src/org/labkey/experiment/ExperimentModule.java | 8 ++++++++ .../org/labkey/experiment/api/ExpMaterialTableImpl.java | 4 +++- 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/experiment/src/org/labkey/experiment/ExperimentModule.java b/experiment/src/org/labkey/experiment/ExperimentModule.java index 588cf576a52..15f7e171368 100644 --- a/experiment/src/org/labkey/experiment/ExperimentModule.java +++ b/experiment/src/org/labkey/experiment/ExperimentModule.java @@ -153,6 +153,9 @@ public class ExperimentModule extends SpringModule implements SearchService.Docu public static final String EXPERIMENT_RUN_WEB_PART_NAME = "Experiment Runs"; + public static final String EXPERIMENTAL_LINEAGE_PARENT_LOOKUP = "lineage-parent-lookup"; + + @Override public String getName() { @@ -213,6 +216,11 @@ protected void init() AdminConsole.addExperimentalFeatureFlag(AppProps.EXPERIMENTAL_RESOLVE_PROPERTY_URI_COLUMNS, "Resolve property URIs as columns on experiment tables", "If a column is not found on an experiment table, attempt to resolve the column name as a Property URI and add it as a property column", false); + AdminConsole.addExperimentalFeatureFlag(EXPERIMENTAL_LINEAGE_PARENT_LOOKUP, + "Expose auto-generated lineage lookup columns in SampleType tables", + "Optimizes 'join' to parent samples/dataclass objects, when relationship is unique (one related row in parent table).", + false); + RoleManager.registerPermission(new DesignVocabularyPermission(), true); AttachmentService.get().registerAttachmentType(ExpRunAttachmentType.get()); diff --git a/experiment/src/org/labkey/experiment/api/ExpMaterialTableImpl.java b/experiment/src/org/labkey/experiment/api/ExpMaterialTableImpl.java index 3a17da594e9..ec314a38123 100644 --- a/experiment/src/org/labkey/experiment/api/ExpMaterialTableImpl.java +++ b/experiment/src/org/labkey/experiment/api/ExpMaterialTableImpl.java @@ -69,6 +69,7 @@ import org.labkey.api.security.permissions.Permission; import org.labkey.api.security.permissions.ReadPermission; import org.labkey.api.security.permissions.UpdatePermission; +import org.labkey.api.settings.AppProps; import org.labkey.api.util.PageFlowUtil; import org.labkey.api.util.Pair; import org.labkey.api.util.StringExpression; @@ -76,6 +77,7 @@ import org.labkey.api.view.ViewContext; import org.labkey.experiment.ExpDataIterators; import org.labkey.experiment.ExpDataIterators.AliasDataIteratorBuilder; +import org.labkey.experiment.ExperimentModule; import org.labkey.experiment.controllers.exp.ExperimentController; import java.io.IOException; @@ -671,7 +673,7 @@ protected ContainerFilter getLookupContainerFilter() setDefaultVisibleColumns(defaultCols); - if (null != _ss) + if (null != _ss && AppProps.getInstance().isExperimentalFeatureEnabled(ExperimentModule.EXPERIMENTAL_LINEAGE_PARENT_LOOKUP)) { MutableColumnInfo lineageLookup = ClosureQueryHelper.createLineageLookupColumnInfo("Parent Lookups", this, _rootTable.getColumn("rowid"), _ss); addColumn(lineageLookup); From 2f1ce78790bf744e49c875491786741ae3cb659e Mon Sep 17 00:00:00 2001 From: Matthew Bellew Date: Thu, 17 Mar 2022 14:50:45 -0700 Subject: [PATCH 8/9] some CR cleanup --- .../experiment/api/ClosureQueryHelper.java | 9 ------ .../experiment/api/ExperimentServiceImpl.java | 29 ++++++++----------- 2 files changed, 12 insertions(+), 26 deletions(-) diff --git a/experiment/src/org/labkey/experiment/api/ClosureQueryHelper.java b/experiment/src/org/labkey/experiment/api/ClosureQueryHelper.java index 13cb438f250..3ceae2552ea 100644 --- a/experiment/src/org/labkey/experiment/api/ClosureQueryHelper.java +++ b/experiment/src/org/labkey/experiment/api/ClosureQueryHelper.java @@ -279,14 +279,6 @@ public static void invalidateAll() } -// public static void invalidateCpasType(String sourceTypeLsid) -// { -// var counter = invalidators.get(sourceTypeLsid); -// if (null != counter) -// counter.incrementAndGet(); -// } - - public static void invalidateMaterialsForRun(String sourceTypeLsid, int runId) { var tx = getScope().getCurrentTransaction(); @@ -392,7 +384,6 @@ private static DbScope getScope() public static MutableColumnInfo createLineageLookupColumnInfo(String columnName, FilteredTable parent, ColumnInfo rowid, ExpObject source) { - //MutableColumnInfo wrappedRowId = wrapColumn("LineageLookupTypes", _rootTable.getColumn("rowid")); MutableColumnInfo wrappedRowId = parent.wrapColumn(columnName, rowid); wrappedRowId.setIsUnselectable(true); wrappedRowId.setReadOnly(true); diff --git a/experiment/src/org/labkey/experiment/api/ExperimentServiceImpl.java b/experiment/src/org/labkey/experiment/api/ExperimentServiceImpl.java index 1cdf4daaf8f..a023d6318ac 100644 --- a/experiment/src/org/labkey/experiment/api/ExperimentServiceImpl.java +++ b/experiment/src/org/labkey/experiment/api/ExperimentServiceImpl.java @@ -2985,28 +2985,28 @@ public void syncRunEdges(Collection runs) class SyncRunEdges { - final int _runId; - final Integer _runObjectId; - final String _runLsid; - final Container _runContainer; + final int runId; + Integer runObjectId; + final String runLsid; + final Container runContainer; boolean deleteFirst = true; boolean verifyEdgesNoInsert=false; boolean invalidateClosureCache=true; SyncRunEdges(ExpRun run) { - this._runId = run.getRowId(); - this._runObjectId = run.getObjectId(); - this._runLsid = run.getLSID(); - this._runContainer = run.getContainer(); + this.runId = run.getRowId(); + this.runObjectId = run.getObjectId(); + this.runLsid = run.getLSID(); + this.runContainer = run.getContainer(); } SyncRunEdges(int runId, Integer runObjectId, String runLsid, Container runContainer) { - this._runId = runId; - this._runObjectId = runObjectId; - this._runLsid = runLsid; - this._runContainer = runContainer; + this.runId = runId; + this.runObjectId = runObjectId; + this.runLsid = runLsid; + this.runContainer = runContainer; } SyncRunEdges deleteFirst(boolean d) @@ -3028,11 +3028,6 @@ SyncRunEdges invalidateClosureCache(boolean i) } void sync(@Nullable Map cpasTypeToObjectId) - { - syncRunEdges(_runId, _runObjectId, _runLsid, _runContainer, deleteFirst, verifyEdgesNoInsert, invalidateClosureCache, cpasTypeToObjectId); - } - - private void syncRunEdges(int runId, Integer runObjectId, String runLsid, Container runContainer, boolean deleteFirst, boolean verifyEdgesNoInsert, boolean invalidateClosureCache, @Nullable Map cpasTypeToObjectId) { // don't do any updates if we are just verifying if (verifyEdgesNoInsert) From 96d1f8d7fc2106ab30ff595e20b58d72b87ecb9b Mon Sep 17 00:00:00 2001 From: Matthew Bellew Date: Mon, 21 Mar 2022 12:44:30 -0700 Subject: [PATCH 9/9] dataclass lookup placeholder name for lookup column --- .../experiment/api/ClosureQueryHelper.java | 23 ++++++++++++------- .../experiment/api/ExpMaterialTableImpl.java | 2 +- 2 files changed, 16 insertions(+), 9 deletions(-) diff --git a/experiment/src/org/labkey/experiment/api/ClosureQueryHelper.java b/experiment/src/org/labkey/experiment/api/ClosureQueryHelper.java index 3ceae2552ea..acf94f9cd3b 100644 --- a/experiment/src/org/labkey/experiment/api/ClosureQueryHelper.java +++ b/experiment/src/org/labkey/experiment/api/ClosureQueryHelper.java @@ -26,6 +26,7 @@ import org.labkey.api.query.FieldKey; import org.labkey.api.query.FilteredTable; import org.labkey.api.query.QueryForeignKey; +import org.labkey.api.query.SchemaKey; import org.labkey.api.query.UserSchema; import org.labkey.api.security.User; import org.labkey.api.util.HeartBeat; @@ -85,7 +86,7 @@ SELECT Start_, CASE WHEN COUNT(*) = 1 THEN MIN(rowId) ELSE -1 * COUNT(*) END AS /*INTO*/ FROM ( SELECT Start_, End_, - COALESCE(material.rowid, dataclass.rowid) as rowId, + COALESCE(material.rowid, data.rowid) as rowId, COALESCE('m' || CAST(materialsource.rowid AS VARCHAR), 'd' || CAST(dataclass.rowid AS VARCHAR)) as targetId FROM CTE_ LEFT OUTER JOIN exp.material ON End_ = material.objectId LEFT OUTER JOIN exp.materialsource ON material.cpasType = materialsource.lsid @@ -118,7 +119,7 @@ SELECT Start_, CASE WHEN COUNT(*) = 1 THEN MIN(rowId) ELSE -1 * COUNT(*) END AS /*INTO*/ FROM ( SELECT Start_, End_, - COALESCE(material.rowid, dataclass.rowid) as rowId, + COALESCE(material.rowid, data.rowid) as rowId, COALESCE('m' + CAST(materialsource.rowid AS VARCHAR), 'd' + CAST(dataclass.rowid AS VARCHAR)) as targetId FROM CTE_ LEFT OUTER JOIN exp.material ON End_ = material.objectId LEFT OUTER JOIN exp.materialsource ON material.cpasType = materialsource.lsid @@ -171,7 +172,8 @@ static MutableColumnInfo createLineageLookupColumn(final ColumnInfo fkRowId, Exp if (!(target instanceof ExpSampleType) && !(target instanceof ExpDataClass)) throw new IllegalStateException(); - final TableType sourceType = source instanceof ExpSampleType ss ? TableType.SampleType : TableType.DataClass; + final TableType sourceType = source instanceof ExpSampleType ? TableType.SampleType : TableType.DataClass; + final TableType targetType = target instanceof ExpSampleType ? TableType.SampleType : TableType.DataClass; TableInfo parentTable = fkRowId.getParentTable(); var ret = new ExprColumn(parentTable, target.getName(), new SQLFragment("#ERROR#"), JdbcType.INTEGER) @@ -188,7 +190,10 @@ public SQLFragment getValueSql(String tableAlias) }; ret.setLabel(target.getName()); UserSchema schema = Objects.requireNonNull(parentTable.getUserSchema()); - ret.setFk(new QueryForeignKey.Builder(schema, parentTable.getContainerFilter()).table(target.getName())); + var qfk = new QueryForeignKey.Builder(schema, parentTable.getContainerFilter()).table(target.getName()).key("rowid"); + if (sourceType != targetType) + qfk.schema(targetType.schemaKey); + ret.setFk(qfk); return ret; } @@ -380,7 +385,7 @@ private static DbScope getScope() /* * Code to create the lineage parent lookup column and intermediate lookups that use ClosureQueryHelper - */ + */ public static MutableColumnInfo createLineageLookupColumnInfo(String columnName, FilteredTable parent, ColumnInfo rowid, ExpObject source) { @@ -427,7 +432,7 @@ public StringExpression getURL(ColumnInfo parent) enum TableType { - SampleType("Materials") + SampleType("Materials", SchemaKey.fromParts("exp","materials") ) { @Override Collection getInstances(Container c, User u) @@ -440,7 +445,7 @@ ExpObject getInstance(Container c, User u, String name) return SampleTypeServiceImpl.get().getSampleType(c, u, name); } }, - DataClass("Data") + DataClass("Data", SchemaKey.fromParts("exp","data") ) { @Override Collection getInstances(Container c, User u) @@ -455,10 +460,12 @@ ExpObject getInstance(Container c, User u, String name) }; final String lookupName; + final SchemaKey schemaKey; - TableType(String lookupName) + TableType(String lookupName, SchemaKey schemaKey) { this.lookupName = lookupName; + this.schemaKey = schemaKey; } abstract Collection getInstances(Container c, User u); diff --git a/experiment/src/org/labkey/experiment/api/ExpMaterialTableImpl.java b/experiment/src/org/labkey/experiment/api/ExpMaterialTableImpl.java index ec314a38123..9001ff89fde 100644 --- a/experiment/src/org/labkey/experiment/api/ExpMaterialTableImpl.java +++ b/experiment/src/org/labkey/experiment/api/ExpMaterialTableImpl.java @@ -675,7 +675,7 @@ protected ContainerFilter getLookupContainerFilter() if (null != _ss && AppProps.getInstance().isExperimentalFeatureEnabled(ExperimentModule.EXPERIMENTAL_LINEAGE_PARENT_LOOKUP)) { - MutableColumnInfo lineageLookup = ClosureQueryHelper.createLineageLookupColumnInfo("Parent Lookups", this, _rootTable.getColumn("rowid"), _ss); + MutableColumnInfo lineageLookup = ClosureQueryHelper.createLineageLookupColumnInfo("Ancestor Lookups Placeholder", this, _rootTable.getColumn("rowid"), _ss); addColumn(lineageLookup); } }