Deaths, births and arrival ETL updates. Add lookup sets

nirc_ehr/resources/data/country.tsv

@@ -0,0 +1,9 @@
+value	title	category
+1	St. Kitts	2
+2	Italy	2
+3	Japan	2
+4	"U.S.A."	1
+5	Philippines	2
+6	Indonesia	2
+7	China	2
+8	Mauritius	2

nirc_ehr/resources/data/country_category.tsv

@@ -0,0 +1,3 @@
+value	title
+1	Domestic
+2	Foreign

nirc_ehr/resources/data/lookup_sets.tsv

@@ -0,0 +1,3 @@
+setname	label	keyfield	titleColumn
+country	Country	value	title
+country_category	Country Category	value	title

nirc_ehr/resources/etls/birth.xml

@@ -15,6 +15,10 @@
         </transform>
     </transforms>
 
+    <incrementalFilter className="ModifiedSinceFilterStrategy" timestampColumnName="modified" >
+        <deletedRowsSource schemaName="dbo" queryName="q_births_delete" timestampColumnName="modified"
+                           deletedSourceKeyColumnName="Id" targetKeyColumnName="Id"/>
+    </incrementalFilter>
     <schedule>
         <cron expression="0 0 20 * * ?" /> <!--Run at 8PM every day-->
     </schedule>

nirc_ehr/resources/etls/deaths.xml

@@ -15,6 +15,10 @@
         </transform>
     </transforms>
 
+    <incrementalFilter className="ModifiedSinceFilterStrategy" timestampColumnName="modified" >
+        <deletedRowsSource schemaName="dbo" queryName="q_deaths_delete" timestampColumnName="modified"
+                           deletedSourceKeyColumnName="Id" targetKeyColumnName="Id"/>
+    </incrementalFilter>
     <schedule>
         <cron expression="0 0 20 * * ?" /> <!--Run at 8PM every day-->
     </schedule>

nirc_ehr/resources/queries/dbo/q_arrival.sql

@@ -1,32 +1,4 @@
-
-        SELECT 'ALT' || alt.ALTERNATE_ID   AS objectid,
-               anm.ANIMAL_ID_NUMBER AS Id,
-               'unknown'      		AS performedby,
-               COALESCE(MAX(CAST(adt.CHANGE_DATETIME AS TIMESTAMP)), to_date('01/01/1970' ,'MM/DD/YYYY')) AS modified,
-               anm.BIRTH_DATE       AS birth,
-               anm.GENDER_ID        AS gender,
-               anm.SSB_ID.SPECIES_ID           AS species,
-               NULL                 AS eventDate,
-               alt.DESCRIPTION      AS acqDateText,
-               'Alternate'          AS source
-        FROM  ALTERNATE alt
-        LEFT JOIN ANIMAL anm ON alt.ANIMAL_ID = anm.ANIMAL_ID
-        LEFT JOIN AUDIT_TRAIL adt ON alt.ALTERNATE_ID = CAST(substring(PRIMARY_KEY_VALUES, length('Alternate_ID = ')) AS INTEGER)
-            AND adt.TABLE_NAME = 'ALTERNATE'
-        WHERE alt.ALTERNATE_TYPE_ID = 7 -- 'Dam/Sire/Acq'
-        AND anm.ANIMAL_ID_NUMBER NOT LIKE 'A%' -- Animal born in centers are pre-appended with A's (in animal Id)
-        AND alt.DESCRIPTION IS NOT NULL
-GROUP BY alt.ALTERNATE_ID,
-         anm.ANIMAL_ID_NUMBER,
-         anm.BIRTH_DATE,
-         anm.GENDER_ID,
-         anm.SSB_ID.SPECIES_ID,
-         alt.NAME,
-         alt.DESCRIPTION
-
-UNION
-
-SELECT '' || anmEvt.ANIMAL_EVENT_ID   AS objectid,
+SELECT anmEvt.ANIMAL_EVENT_ID   AS objectid,
        anm.ANIMAL_ID_NUMBER  AS Id,
        (CASE
             WHEN (anmEvt.STAFF_ID.STAFF_FIRST_NAME IS NULL OR anmEvt.STAFF_ID.STAFF_LAST_NAME IS NULL) THEN 'unknown'

nirc_ehr/resources/queries/dbo/q_arrival_delete.sql

@@ -1,12 +1,3 @@
-SELECT 'ALT' || substring(adt.PRIMARY_KEY_VALUES, length('Alternate_ID = ')) AS objectid,
-       CAST(adt.CHANGE_DATETIME AS TIMESTAMP)                            AS modified,
-       adt.REFERENCE
-FROM AUDIT_TRAIL adt
-WHERE adt.TABLE_NAME = 'ALTERNATE' AND
-        adt.COLUMN_NAME = 'DELETE'
-
-UNION
-
 SELECT substring(adt.PRIMARY_KEY_VALUES, length('ANIMAL_EVENT_ID = ')) AS objectid,
        CAST(adt.CHANGE_DATETIME AS TIMESTAMP)                            AS modified,
        adt.REFERENCE

nirc_ehr/resources/queries/dbo/q_births.sql

@@ -1,9 +1,16 @@
 SELECT anm.ANIMAL_ID_NUMBER AS Id,
        anm.BIRTH_DATE       AS birthDate,
        anm.GENDER_ID        AS gender,
-       anm.SSB_ID.SPECIES_ID           AS species
+       anm.SSB_ID.SPECIES_ID           AS species,
+       COALESCE(MAX(CAST(adt.CHANGE_DATETIME AS TIMESTAMP)), to_date('01/01/1970' ,'MM/DD/YYYY')) AS modified
 FROM Animal anm
 LEFT JOIN ALTERNATE alt ON alt.ANIMAL_ID = anm.ANIMAL_ID
+LEFT JOIN AUDIT_TRAIL adt ON anm.ANIMAL_ID_NUMBER = substring(PRIMARY_KEY_VALUES, length('ANIMAL_ID = '))
+    AND adt.TABLE_NAME = 'ANIMAL'
 WHERE alt.ALTERNATE_TYPE_ID = 7 -- 'Dam/Sire/Acq'
 AND anm.ANIMAL_ID_NUMBER LIKE 'A%' -- Animal born in centers are pre-appended with A's (in animal Id)
-AND alt.DESCRIPTION IS NOT NULL
+AND alt.DESCRIPTION IS NOT NULL
+GROUP BY anm.ANIMAL_ID_NUMBER,
+         anm.BIRTH_DATE,
+         anm.GENDER_ID,
+         anm.SSB_ID.SPECIES_ID

nirc_ehr/resources/queries/dbo/q_births_delete.sql

@@ -0,0 +1,5 @@
+SELECT substring(adt.PRIMARY_KEY_VALUES, length('ANIMAL_ID = ')) AS Id,
+       CAST(adt.CHANGE_DATETIME AS TIMESTAMP)                    AS modified,
+       adt.REFERENCE
+FROM AUDIT_TRAIL adt
+WHERE adt.TABLE_NAME = 'ANIMAL' AND adt.COLUMN_NAME = 'DELETE'

nirc_ehr/resources/queries/dbo/q_deaths.sql

@@ -1,7 +1,13 @@
 
-SELECT ANIMAL_ID_NUMBER AS Id,
-       DEATH_DATE AS deathDate,
-       TERMINATION_REASON_ID as reason
-FROM Animal
-WHERE ANIMAL_DISPOSITION_ID = 4 -- died
-AND TERMINATION_REASON_ID != 10 -- Invalid Id
+SELECT an.ANIMAL_ID_NUMBER AS Id,
+       an.DEATH_DATE AS deathDate,
+       an.TERMINATION_REASON_ID as reason,
+       COALESCE(MAX(CAST(adt.CHANGE_DATETIME AS TIMESTAMP)), to_date('01/01/1970' ,'MM/DD/YYYY')) AS modified
+FROM Animal an
+LEFT JOIN AUDIT_TRAIL adt ON an.ANIMAL_ID_NUMBER = substring(PRIMARY_KEY_VALUES, length('ANIMAL_ID = '))
+    AND adt.TABLE_NAME = 'ANIMAL'
+WHERE an.ANIMAL_DISPOSITION_ID = 4 -- died
+AND an.TERMINATION_REASON_ID != 10 -- Invalid Id
+GROUP BY an.ANIMAL_ID_NUMBER,
+    an.DEATH_DATE,
+    an.TERMINATION_REASON_ID

nirc_ehr/resources/queries/dbo/q_deaths_delete.sql

@@ -0,0 +1,5 @@
+SELECT substring(adt.PRIMARY_KEY_VALUES, length('ANIMAL_ID = ')) AS Id,
+       CAST(adt.CHANGE_DATETIME AS TIMESTAMP)                    AS modified,
+       adt.REFERENCE
+FROM AUDIT_TRAIL adt
+WHERE adt.TABLE_NAME = 'ANIMAL' AND adt.COLUMN_NAME = 'DELETE'

nirc_ehr/resources/queries/ehr_lookups/country.query.xml

@@ -0,0 +1,20 @@
+<query xmlns="http://labkey.org/data/xml/query">
+    <metadata>
+        <tables xmlns="http://labkey.org/data/xml">
+            <table tableName="country" tableDbType="TABLE">
+                <tableTitle>Country</tableTitle>
+                <columns>
+                    <column columnName="category">
+                        <columnTitle>Category</columnTitle>
+                        <fk>
+                            <fkDbSchema>ehr_lookups</fkDbSchema>
+                            <fkTable>country_category</fkTable>
+                            <fkColumnName>value</fkColumnName>
+                            <fkDisplayColumnName>title</fkDisplayColumnName>
+                        </fk>
+                    </column>
+                </columns>
+            </table>
+        </tables>
+    </metadata>
+</query>

nirc_ehr/resources/queries/study/arrival.js

@@ -4,26 +4,7 @@ EHR.Server.TriggerManager.registerHandlerForQuery(EHR.Server.TriggerManager.Even
     // if born at center acq date and birth date is same (row.description contains ACQ date)
     // if brought from outside aqc date and birth date may differ
 
-    if (row.source === 'Alternate' ) {
-        if (row.acqDateText) {
-            let acqDateText = row.acqDateText.split(':')[1]; // format - ACQ: 24 Sep 2008
-            if (!acqDateText) {
-                acqDateText = row.acqDateText.split(';')[1]; // at least one value is there with ';'
-            }
-            if (acqDateText) {
-                let acqDate = new Date(acqDateText.trim());
-                if (acqDate !== row.birth) {
-                    row.date = acqDate;
-                }
-                else {
-                    // investigate to see if any row exists
-                    console.log("born at center - ", row.Id);
-                    row.date = row.birth;
-                }
-            }
-        }
-    }
-    else if (row.source === 'Animal Event' && row.eventDate) {
+    if (row.eventDate) {
         row.date = row.eventDate;
     }
 });

nirc_ehr/resources/schemas/dbscripts/postgresql/nirc_ehr-22.001-22.002.sql

@@ -0,0 +1,3 @@
+SELECT core.executeJavaUpgradeCode('etl;{NIRC_EHR}/arrival;truncate');
+SELECT core.executeJavaUpgradeCode('etl;{NIRC_EHR}/birth;truncate');
+SELECT core.executeJavaUpgradeCode('etl;{NIRC_EHR}/deaths;truncate');

nirc_ehr/resources/views/populateData.html

@@ -36,12 +36,33 @@
             schemaName: 'ehr_lookups',
             queryName: 'ageclass',
             pk: 'value'
+        }, {
+            label: 'Country',
+            populateFn: 'populateFromFile',
+            schemaName: 'ehr_lookups',
+            queryName: 'country',
+            pk: 'value'
+        }, {
+            label: 'Country Category',
+            populateFn: 'populateFromFile',
+            schemaName: 'ehr_lookups',
+            queryName: 'country_category',
+            pk: 'value'
         }];
 
         tables.sort(function(a, b) {
             return a.label.toLowerCase() < b.label.toLowerCase() ? -1 : 1;
         });
 
+        // Keep lookup sets as the first item
+        tables.splice(0, 0, {
+            label: 'Lookup Sets',
+            doSkip: true,
+            populateFn: 'populateLookupSets',
+            schemaName: 'ehr_lookups',
+            queryName: 'lookup_sets'
+        });
+
         var webpart = <%=webpartContext%>;
         Ext4.define('NIRC_EHR.panel.PopulateDataPanel', {
             extend: 'Ext.panel.Panel',
@@ -229,7 +250,27 @@
                 if (this.pendingInserts==0){
                     document.getElementById('msgbox').innerHTML += '<div>Populate Complete</div>';
                 }
-            }
+            },
+
+            populateLookupSets: function () {
+                this.pendingInserts++;
+
+                //records for reports:
+                var config = {
+                    schemaName: 'ehr_lookups',
+                    queryName: 'lookup_sets',
+                    moduleResource: '/data/lookup_sets.tsv',
+                    success: this.onSuccess,
+                    failure: this.onError,
+                    scope: this
+                };
+
+                config.success = function (results, xhr, c) {
+                    console.log('lookup set records inserted');
+                };
+
+                importFile(config);
+            },
         });
 
         Ext4.create('NIRC_EHR.panel.PopulateDataPanel').render(webpart.wrapperDivId);

nirc_ehr/src/org/labkey/nirc_ehr/NIRC_EHRModule.java

@@ -51,7 +51,7 @@ public String getName()
     @Override
     public @Nullable Double getSchemaVersion()
     {
-        return 22.001;
+        return 22.002;
     }
 
     @Override