Merge discvr-24.3 to develop

MccColony/resources/queries/study/demographics/.qview.xml

@@ -26,6 +26,7 @@
         <column name="Id/MostRecentWeight/MostRecentWeightGrams"/>
         <column name="Id/MostRecentWeight/MostRecentWeightDate"/>
         <column name="alternateIds"/>
+        <column name="litterId"/>
         <column name="excludeFromCensus"/>
         <column name="Id/mostRecentObservations/availability::observation">
             <properties>

mGAP/src/org/labkey/mgap/columnTransforms/LuceneIndexTransform.java

@@ -1,14 +1,11 @@
 package org.labkey.mgap.columnTransforms;
 
-import org.apache.commons.io.FileUtils;
 import org.jetbrains.annotations.Nullable;
+import org.labkey.api.jbrowse.JBrowseService;
 import org.labkey.api.pipeline.PipelineJobException;
-import org.labkey.api.sequenceanalysis.run.SimpleScriptWrapper;
 import org.labkey.mgap.etl.EtlQueueManager;
 
 import java.io.File;
-import java.io.IOException;
-import java.util.Arrays;
 
 public class LuceneIndexTransform extends OutputFileTransform
 {
@@ -27,6 +24,7 @@ protected File doFileCopy(File f, File subdir, @Nullable String name) throws Pip
         // NOTE: lucene is a special case since the DB tracks one file, but we need this whole folder:
         File sourceDir = f.getParentFile();
         File targetDir = new File(subdir, "LuceneIndex");
+        JBrowseService.get().clearLuceneCacheEntry(targetDir);
         EtlQueueManager.get().queueRsyncCopy(getContainerUser().getContainer(), sourceDir, targetDir);
 
         return new File(targetDir, sourceDir.getName() + "/" + f.getName());

mGAP/src/org/labkey/mgap/pipeline/mGapSummarizer.java

@@ -174,7 +174,7 @@ private static class FieldData
     public static void filterCodingPotential(Set<String> codingPotential)
     {
         //due to overlapping transcripts, this is often added.  remove these less-specific terms in order
-        for (String type : Arrays.asList("intragenic_variant", "non_coding_transcript_variant", "intron_variant"))
+        for (String type : Arrays.asList("custom", "intergenic_variant", "intragenic_variant"))
         {
             if (codingPotential.size() > 1)
             {
@@ -184,9 +184,8 @@ public static void filterCodingPotential(Set<String> codingPotential)
 
         if (codingPotential.contains("synonymous_variant") || codingPotential.contains("missense_variant"))
         {
+            codingPotential.remove("intergenic_variant");
             codingPotential.remove("intragenic_variant");
-            codingPotential.remove("non_coding_transcript_variant");
-            codingPotential.remove("intron_variant");
         }
     }
 

mcc/resources/etls/mcc.xml

@@ -38,7 +38,7 @@
                     <column>objectid</column>
                 </sourceColumns>
             </source>
-            <destination schemaName="study" queryName="Demographics" targetOption="truncate" bulkLoad="true">
+            <destination schemaName="study" queryName="Demographics" targetOption="truncate" bulkLoad="true" batchSize="5000">
                 <alternateKeys>
                     <column name="objectid"/>
                 </alternateKeys>
@@ -56,7 +56,7 @@
                     <column>objectid</column>
                 </sourceColumns>
             </source>
-            <destination schemaName="study" queryName="kinship" targetOption="truncate" bulkLoad="true">
+            <destination schemaName="study" queryName="kinship" targetOption="truncate" bulkLoad="true" batchSize="5000">
                 <alternateKeys>
                     <column name="objectid"/>
                 </alternateKeys>
@@ -72,7 +72,7 @@
                     <column>objectid</column>
                 </sourceColumns>
             </source>
-            <destination schemaName="study" queryName="weight" targetOption="truncate" bulkLoad="true">
+            <destination schemaName="study" queryName="weight" targetOption="truncate" bulkLoad="true" batchSize="5000">
                 <alternateKeys>
                     <column name="objectid"/>
                 </alternateKeys>
@@ -89,7 +89,7 @@
                     <column>objectid</column>
                 </sourceColumns>
             </source>
-            <destination schemaName="study" queryName="genomicDatasets" targetOption="truncate" bulkLoad="true">
+            <destination schemaName="study" queryName="genomicDatasets" targetOption="truncate" bulkLoad="true" batchSize="5000">
                 <alternateKeys>
                     <column name="objectid"/>
                 </alternateKeys>
@@ -106,7 +106,7 @@
                     <column>objectid</column>
                 </sourceColumns>
             </source>
-            <destination schemaName="study" queryName="clinical_observations" targetOption="truncate" bulkLoad="true">
+            <destination schemaName="study" queryName="clinical_observations" targetOption="truncate" bulkLoad="true" batchSize="5000">
                 <alternateKeys>
                     <column name="objectid"/>
                 </alternateKeys>
@@ -127,7 +127,7 @@
                     <column>marmosetsShipped</column>
                 </sourceColumns>
             </source>
-            <destination schemaName="mcc" queryName="census" targetOption="truncate" bulkLoad="true">
+            <destination schemaName="mcc" queryName="census" targetOption="truncate" bulkLoad="true" batchSize="5000">
 
             </destination>
         </transform>

mcc/resources/queries/mcc/aggregatedDemographics.sql

@@ -19,6 +19,7 @@ SELECT
     WHEN d.calculated_status = 'Alive' AND (SELECT COUNT(f.flag.value) as total FROM "/data/Colonies/SNPRC/".study.flags f WHERE f.Id = d.Id AND f.isActive = true) > 0 THEN true
     ELSE false
   END as u24_status,
+  d.litterId,
   d.Id.mostRecentDeparture.mostRecentDeparture,
   o.availability,
   o.current_housing_status,
@@ -67,6 +68,7 @@ SELECT
     WHEN d.calculated_status = 'Alive' AND (SELECT COUNT(f.flag.value) as total FROM "/data/Colonies/WNPRC/".study.flags f WHERE f.Id = d.Id AND f.isActive = true) > 0 THEN true
     ELSE false
   END as u24_status,
+  d.litterId,
   d.Id.mostRecentDeparture.mostRecentDeparture,
   o.availability,
   o.current_housing_status,
@@ -112,6 +114,7 @@ SELECT
     d.objectid,
     d.calculated_status,
     d.u24_status,
+    d.litterId,
     d.Id.mostRecentDeparture.mostRecentDeparture,
     o.availability,
     o.current_housing_status,
@@ -160,6 +163,7 @@ SELECT
     d.objectid,
     d.calculated_status,
     d.u24_status,
+    d.litterId,
     d.Id.mostRecentDeparture.mostRecentDeparture,
     o.availability,
     o.current_housing_status,

mcc/resources/queries/study/demographics.js

@@ -111,6 +111,13 @@ function onUpsert(helper, scriptErrors, row, oldRow){
         }
     }
 
+    if (oldRow && oldRow.Id) {
+        var existingId = triggerHelper.getMccAlias(oldRow.Id);
+        if (existingId) {
+            idToMccAlias[row.Id] = existingId;
+        }
+    }
+
     if (!row.date) {
         row.date = new Date();
     }

mcc/resources/queries/study/demographics.query.xml

@@ -9,7 +9,7 @@
                 <deleteUrl />
                 <columns>
                     <column columnName="Id">
-                        <url></url>
+                        <url />
                     </column>
                     <column columnName="date">
                         <isHidden>true</isHidden>
@@ -121,6 +121,13 @@
                     <column columnName="alternateIds">
                         <columnTitle>Alternate IDs</columnTitle>
                     </column>
+                    <column columnName="litterId">
+                        <columnTitle>Litter ID</columnTitle>
+                        <url>/query/executeQuery.view?schemaName=study&amp;
+                            query.queryName=Demographics&amp;
+                            query.litterId~eq=${litterId}
+                        </url>
+                    </column>
                     <column columnName="remark">
 
                     </column>

mcc/resources/queries/study/demographics/.qview.xml

@@ -36,6 +36,7 @@
         <column name="usage_current"/>
         <column name="usage_future"/>
         <column name="date_of_observations"/>
+        <column name="litterId"/>
     </columns>
     <sorts>
         <sort column="Id" descending="false"/>

mcc/resources/queries/study/demographicsLittermates.query.xml

@@ -0,0 +1,23 @@
+<query xmlns="http://labkey.org/data/xml/query">
+    <description>Littermates of Any Animal</description>
+    <metadata>
+        <tables xmlns="http://labkey.org/data/xml">
+            <table tableName="demographicsLittermates" tableDbType="NOT_IN_DB">
+                <description>Littermates of Any Animal</description>
+                <columns>
+                    <column columnName="Id">
+                        <isKeyField>true</isKeyField>
+                        <isHidden>true</isHidden>
+                    </column>
+                    <column columnName="litterId">
+
+                    </column>
+                    <column columnName="litterMates">
+                        <columnTitle>Littermate(s)</columnTitle>
+                    </column>
+                </columns>
+                <titleColumn>Relationship</titleColumn>
+            </table>
+        </tables>
+    </metadata>
+</query>

mcc/resources/queries/study/demographicsLittermates.sql

@@ -0,0 +1,16 @@
+SELECT
+
+d1.Id,
+d1.litterId,
+
+GROUP_CONCAT(d2.Id, ',') as litterMates,
+
+FROM study.Demographics d1
+
+JOIN study.Demographics d2 ON (d1.litterId = d2.litterId AND d1.id != d2.id)
+
+WHERE
+    d1.qcstate.publicdata = true AND
+    d2.qcstate.publicdata = true
+
+GROUP BY d1.Id, d1.litterId

mcc/resources/referenceStudy/study/datasets/datasets_metadata.xml

@@ -685,6 +685,9 @@
             <column columnName="excludeFromCensus">
                 <datatype>boolean</datatype>
             </column>
+            <column columnName="litterId">
+                <datatype>varchar</datatype>
+            </column>
         </columns>
         <tableTitle>Demographics</tableTitle>
     </table>