Lineage lookups

api/src/org/labkey/api/data/MaterializedQueryHelper.java

@@ -56,12 +56,14 @@ private class Materialized
         private final long _created;
         private final String _cacheKey;
         private final String _fromSql;
+        private final String _tableName;
         private final ArrayList<Invalidator> _invalidators = new ArrayList<>(3);
 
-        Materialized(String cacheKey, long created, String sql)
+        Materialized(String tableName, String cacheKey, long created, String sql)
         {
             _created = created;
             _cacheKey = cacheKey;
+            _tableName = tableName;
             _fromSql = sql;
         }
 
@@ -191,11 +193,11 @@ private String makeKey(DbScope.Transaction t, Container c)
     private final String _prefix;
     private final DbScope _scope;
     private final SQLFragment _selectQuery;
+    private final boolean _isSelectIntoSql;
     private final SQLFragment _uptodateQuery;
     private final Supplier<String> _supplier;
     private final List<String> _indexes = new ArrayList<>();
     private final long _maxTimeToCache;
-    private final boolean _perContainer;
     private final LinkedHashMap<String, Materialized> _map = new LinkedHashMap<String,Materialized>()
     {
         @Override
@@ -214,19 +216,17 @@ protected boolean removeEldestEntry(Map.Entry<String, Materialized> eldest)
     private boolean _closed = false;
 
     private MaterializedQueryHelper(String prefix, DbScope scope, SQLFragment select, @Nullable SQLFragment uptodate, Supplier<String> supplier, @Nullable Collection<String> indexes, long maxTimeToCache,
-                                    boolean perContainer)
+                                    boolean isSelectIntoSql)
     {
         _prefix = StringUtils.defaultString(prefix,"mat");
         _scope = scope;
         _selectQuery = select;
         _uptodateQuery = uptodate;
         _supplier = supplier;
         _maxTimeToCache = maxTimeToCache;
-        _perContainer = perContainer;
         if (null != indexes)
             _indexes.addAll(indexes);
-        if (perContainer)
-            throw new UnsupportedOperationException("NYI");
+        _isSelectIntoSql = isSelectIntoSql;
         assert MemTracker.get().put(this);
     }
 
@@ -303,57 +303,50 @@ public SQLFragment getFromSql(String tableAlias, Container c)
     {
         if (null == _selectQuery)
             throw new IllegalStateException("Must specify source query in constructor or in getFromSql()");
-        return getFromSql(_selectQuery, tableAlias, c);
+        return getFromSql(_selectQuery, _isSelectIntoSql, tableAlias, c);
     }
 
-    /* NOTE: we do not want to hold synchronized(this) while doing any SQL operations */
-    public SQLFragment getFromSql(@NotNull SQLFragment selectQuery, String tableAlias, Container c)
+
+    public boolean isCached(Container c)
     {
-        Materialized materialized = null;
-        final String txCacheKey = makeKey(_scope.getCurrentTransaction(), c);
-        final long now = HeartBeat.currentTimeMillis();
+        if (null == _selectQuery)
+            throw new IllegalStateException("Must specify source query in constructor or in getFromSql()");
+        return null != getMaterialized(makeKey(_scope.getCurrentTransaction(), c));
+    }
 
-        synchronized (this)
-        {
-            if (_closed)
-                throw new IllegalStateException();
-            if (null != c)
-                throw new UnsupportedOperationException();
 
-            _countGetFromSql.incrementAndGet();
+    public void upsert(SQLFragment sqlf)
+    {
+        String txCacheKey = makeKey(_scope.getCurrentTransaction(), null);
+        Materialized m = getMaterialized(txCacheKey);
+        if (null == m)
+            return;
+        String sql = sqlf.getSQL().replace("${NAME}", m._tableName);
+        List<Object> params = sqlf.getParams();
+        new SqlExecutor(_scope).execute(new SQLFragment(sql,params));
+    }
 
-            if (_scope.isTransactionActive())
-                materialized = _map.get(txCacheKey);
 
-            if (null == materialized)
-                materialized = _map.get(makeKey(null, c));
-        }
+    /* used by FLow directly for some reason */
+    public SQLFragment getFromSql(@NotNull SQLFragment selectQuery, String tableAlias, Container c)
+    {
+        return getFromSql(selectQuery, false, tableAlias, c);
+    }
 
-        if (null != materialized)
-        {
-            boolean replace = false;
-            for (Invalidator i : materialized._invalidators)
-            {
-                CacheCheck cc = i.checkValid(materialized._created);
-                if (cc != CacheCheck.OK)
-                    replace = true;
-            }
-            if (replace)
-            {
-                synchronized (this)
-                {
-                    _map.remove(materialized._cacheKey);
-                    materialized = null;
-                }
-            }
-        }
+    /* NOTE: we do not want to hold synchronized(this) while doing any SQL operations */
+    public SQLFragment getFromSql(@NotNull SQLFragment selectQuery, boolean isSelectInto, String tableAlias, Container c)
+    {
+        final String txCacheKey = makeKey(_scope.getCurrentTransaction(), c);
+        final long now = HeartBeat.currentTimeMillis();
+
+        Materialized materialized = getMaterialized(txCacheKey);
 
         if (null == materialized)
         {
             _countSelectInto.incrementAndGet();
             DbSchema temp = DbSchema.getTemp();
             String name = _prefix + "_" + GUID.makeHash();
-            materialized = new Materialized(txCacheKey, now, "\"" + temp.getName() + "\".\"" + name + "\"");
+            materialized = new Materialized(name, txCacheKey, now, "\"" + temp.getName() + "\".\"" + name + "\"");
             materialized.addMaxTimeToCache(_maxTimeToCache);
             materialized.addUpToDateQuery(_uptodateQuery);
             materialized.addInvalidator(_supplier);
@@ -362,9 +355,19 @@ public SQLFragment getFromSql(@NotNull SQLFragment selectQuery, String tableAlia
 
             TempTableTracker.track(name, materialized);
 
-            SQLFragment selectInto = new SQLFragment("SELECT * INTO \"" + temp.getName() + "\".\"" + name + "\"\nFROM (\n");
-            selectInto.append(selectQuery);
-            selectInto.append("\n) _sql_");
+            SQLFragment selectInto;
+            if (isSelectInto)
+            {
+                String sql = selectQuery.getSQL().replace("${NAME}", name);
+                List<Object> params = selectQuery.getParams();
+                selectInto = new SQLFragment(sql,params);
+            }
+            else
+            {
+                selectInto = new SQLFragment("SELECT * INTO \"" + temp.getName() + "\".\"" + name + "\"\nFROM (\n");
+                selectInto.append(selectQuery);
+                selectInto.append("\n) _sql_");
+            }
             new SqlExecutor(_scope).execute(selectInto);
 
             try (var ignored = SpringActionController.ignoreSqlUpdates())
@@ -394,6 +397,53 @@ public SQLFragment getFromSql(@NotNull SQLFragment selectQuery, String tableAlia
         return sqlf;
     }
 
+    @Nullable
+    private Materialized getMaterialized(String txCacheKey)
+    {
+        Materialized materialized = null;
+
+        synchronized (this)
+        {
+            if (_closed)
+                throw new IllegalStateException();
+
+            _countGetFromSql.incrementAndGet();
+
+            if (_scope.isTransactionActive())
+                materialized = _map.get(txCacheKey);
+
+            if (null == materialized)
+                materialized = _map.get(makeKey(null, null));
+        }
+
+        if (null != materialized)
+        {
+            boolean replace = false;
+            for (Invalidator i : materialized._invalidators)
+            {
+                CacheCheck cc = i.checkValid(materialized._created);
+                if (cc != CacheCheck.OK)
+                    replace = true;
+            }
+            if (replace)
+            {
+                synchronized (this)
+                {
+                    _map.remove(materialized._cacheKey);
+                    materialized = null;
+                }
+            }
+        }
+        return materialized;
+    }
+
+
+    /* Do incremental update to existing cached data.  There is no provision for deleting rows. */
+    public void upsert()
+    {
+
+    }
+
 
     /**
      *  To be consistent with CacheManager maxTimeToCache==0 means UNLIMITED, so we use maxTimeToCache==-1 to mean no caching, just materialize and return
@@ -427,6 +477,7 @@ public static class Builder implements org.labkey.api.data.Builder<MaterializedQ
         private final DbScope _scope;
         private final SQLFragment _select;
 
+        private boolean _isSelectInto = false;
         private long _max = CacheManager.UNLIMITED;
         private SQLFragment _uptodate = null;
         private Supplier<String> _supplier = null;
@@ -439,6 +490,13 @@ public Builder(String prefix, DbScope scope, SQLFragment select)
             _select = select;
         }
 
+        /** This property indicates that the SQLFragment is formatted as a SELECT INTO query (rather than a simple SELECT) */
+        public Builder setIsSelectInto(boolean b)
+        {
+            _isSelectInto = b;
+            return this;
+        }
+
         public Builder upToDateSql(SQLFragment uptodate)
         {
             _uptodate = uptodate;
@@ -466,7 +524,7 @@ public Builder addIndex(String index)
         @Override
         public MaterializedQueryHelper build()
         {
-            return new MaterializedQueryHelper(_prefix, _scope, _select, _uptodate, _supplier, _indexes, _max, false);
+            return new MaterializedQueryHelper(_prefix, _scope, _select, _uptodate, _supplier, _indexes, _max, _isSelectInto);
         }
     }
 

api/src/org/labkey/api/data/TableSelector.java

@@ -42,6 +42,7 @@
 import java.util.List;
 import java.util.Map;
 import java.util.Set;
+import java.util.stream.Collectors;
 import java.util.stream.Stream;
 
 public class TableSelector extends SqlExecutingSelector<TableSelector.TableSqlFactory, TableSelector> implements ResultsFactory
@@ -129,7 +130,9 @@ private static Collection<ColumnInfo> columnInfosList(@NotNull TableInfo table,
 
         if (select == ALL_COLUMNS)
         {
-            selectColumns = table.getColumns();
+            selectColumns = table.getColumns().stream()
+                .filter(columnInfo -> !columnInfo.isUnselectable())
+                .collect(Collectors.toList());
         }
         else
         {

experiment/src/org/labkey/experiment/ExperimentModule.java

@@ -153,6 +153,9 @@ public class ExperimentModule extends SpringModule implements SearchService.Docu
 
     public static final String EXPERIMENT_RUN_WEB_PART_NAME = "Experiment Runs";
 
+    public static final String EXPERIMENTAL_LINEAGE_PARENT_LOOKUP = "lineage-parent-lookup";
+
+
     @Override
     public String getName()
     {
@@ -213,6 +216,11 @@ protected void init()
         AdminConsole.addExperimentalFeatureFlag(AppProps.EXPERIMENTAL_RESOLVE_PROPERTY_URI_COLUMNS, "Resolve property URIs as columns on experiment tables",
                 "If a column is not found on an experiment table, attempt to resolve the column name as a Property URI and add it as a property column", false);
 
+        AdminConsole.addExperimentalFeatureFlag(EXPERIMENTAL_LINEAGE_PARENT_LOOKUP,
+                "Expose auto-generated lineage lookup columns in SampleType tables",
+                "Optimizes 'join' to parent samples/dataclass objects, when relationship is unique (one related row in parent table).",
+                false);
+
         RoleManager.registerPermission(new DesignVocabularyPermission(), true);
 
         AttachmentService.get().registerAttachmentType(ExpRunAttachmentType.get());