Breeder and Clinical Remarks ETL and reporting

.gitignore

@@ -1,6 +1,10 @@
 .gradle
 build/
 
+# Log Files
+nirc_ehr/resources/referenceStudy/etlLogs/
+nirc_ehr/resources/referenceStudy/*.log
+
 # Ignore Gradle GUI config
 gradle-app.setting
 

nirc_ehr/resources/data/reports.tsv

@@ -11,6 +11,8 @@ currentBlood	Clinical	js	Current Blood	true		study	currentBlood			date	false	fal
 bloodDraws	Clinical	query	Blood Draws	TRUE		study	blood			date	FALSE	FALSE		qcstate/publicdata	This report displays blood draw data for the selected animal
 obs	Clinical	query	Observations	TRUE		study	obs			date	FALSE	FALSE		qcstate/publicdata	This report displays observations for the selected animal
 pairings	Colony Management	query	Pairings	TRUE		study	pairings			date	FALSE	FALSE		qcstate/publicdata	This report displays pairings for the selected animal
+breeder	Colony Management	query	Breeder	TRUE		study	breeder			date	FALSE	FALSE		qcstate/publicdata	This report displays breeding data for the selected animal
+clinremarks	Clinical	query	Clinical Remarks	true		study	Clinical Remarks			date	false	false		qcstate/publicdata	This report contains the clinical remarks entered about each animal
 serology	ClinPath	query	Serology	TRUE		study	serology			date	FALSE	FALSE		qcstate/publicdata	This report displays serology data for the selected animal
 vitals	Clinical	query	Vital Signs	TRUE		study	vitals			date	FALSE	FALSE		qcstate/publicdata	This report displays vitals data for the selected animal
 physicalExam	Clinical	query	Physical Exam	TRUE		study	physicalExam			date	FALSE	FALSE		qcstate/publicdata	This report displays physical exam data for the selected animal

nirc_ehr/resources/etls/breeder.xml

@@ -0,0 +1,31 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<etl xmlns="http://labkey.org/etl/xml">
+    <name>Breeder</name>
+    <description>Breeder</description>
+    <transforms>
+        <transform id="step1" type="org.labkey.di.pipeline.TransformTask">
+            <description>Copy to breeder target</description>
+            <source schemaName="dbo" queryName="q_breeder"/>
+            <destination schemaName="study" queryName="breeder"
+                         bulkLoad="true" batchSize="5000" targetOption="merge">
+                <columnTransforms>
+                    <column source="administrationDate" target="date"/>
+                    <column source="performedby" target="performedby" transformClass="org.labkey.di.columnTransforms.UserCreateTransform"/>
+                </columnTransforms>
+            </destination>
+        </transform>
+    </transforms>
+
+    <incrementalFilter className="ModifiedSinceFilterStrategy" timestampColumnName="modified" >
+        <deletedRowsSource schemaName="dbo" queryName="q_breeder_delete" timestampColumnName="modified"
+                           deletedSourceKeyColumnName="objectid" targetKeyColumnName="objectid"/>
+    </incrementalFilter>
+
+    <constants>
+        <column type="varchar" name="transformEmailDomain" value="louisiana.edu"/>
+        <column type="varchar" name="userGroupName" value="NIRC ETL USERS"/>
+    </constants>
+    <schedule>
+        <cron expression="0 0 20 * * ?" /> <!--Run at 8PM every day-->
+    </schedule>
+</etl>

nirc_ehr/resources/etls/clinremarks.xml

@@ -0,0 +1,31 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<etl xmlns="http://labkey.org/etl/xml">
+    <name>Clinical Remarks</name>
+    <description>Comments/Clinical Remarks</description>
+    <transforms>
+        <transform id="step1" type="org.labkey.di.pipeline.TransformTask">
+            <description>Copy to clinremarks target</description>
+            <source schemaName="dbo" queryName="q_clinremarks"/>
+            <destination schemaName="study" queryName="clinremarks"
+                         bulkLoad="true" batchSize="5000" targetOption="merge">
+                <columnTransforms>
+                    <column source="administrationDate" target="date"/>
+                    <column source="performedby" target="performedby" transformClass="org.labkey.di.columnTransforms.UserCreateTransform"/>
+                </columnTransforms>
+            </destination>
+        </transform>
+    </transforms>
+
+    <incrementalFilter className="ModifiedSinceFilterStrategy" timestampColumnName="modified" >
+        <deletedRowsSource schemaName="dbo" queryName="q_clinremarks_delete" timestampColumnName="modified"
+                           deletedSourceKeyColumnName="objectid" targetKeyColumnName="objectid"/>
+    </incrementalFilter>
+
+    <constants>
+        <column type="varchar" name="transformEmailDomain" value="louisiana.edu"/>
+        <column type="varchar" name="userGroupName" value="NIRC ETL USERS"/>
+    </constants>
+    <schedule>
+        <cron expression="0 0 20 * * ?" /> <!--Run at 8PM every day-->
+    </schedule>
+</etl>

nirc_ehr/resources/queries/dbo/q_breeder.sql

@@ -0,0 +1,21 @@
+SELECT anmEvt.ANIMAL_EVENT_ID                                                    AS objectid,
+       anmEvt.ANIMAL_ID                                                          AS Id,
+       CAST(anmEvt.EVENT_DATETIME AS TIMESTAMP)                                  AS administrationDate,
+       (CASE
+            WHEN anmEvt.STAFF_ID.EMAIL_ADDRESS IS NULL THEN 'unknown'
+            ELSE substr(anmEvt.STAFF_ID.EMAIL_ADDRESS, 1,
+                        instr(anmEvt.STAFF_ID.EMAIL_ADDRESS, '@') - 1) END)      AS performedby,
+       anmEvt.EVENT_ID.NAME                                                      AS type,
+       anmEvt.TEXT_RESULT                                                        AS result,
+       anmEvt.ATTACHMENT_PATH                                                    AS attachmentFile,
+       anmCmt.TEXT                                                               AS remark,
+       CAST(COALESCE(adt.CHANGE_DATETIME, anmEvt.CREATED_DATETIME) AS TIMESTAMP) AS modified
+FROM ANIMAL_EVENT anmEvt
+         LEFT JOIN ANIMAL_EVENT_COMMENT anmCmt ON anmEvt.ANIMAL_EVENT_ID = anmCmt.ANIMAL_EVENT_ID
+         LEFT JOIN EVENT_EVENT_GROUP evtEvtGrp ON evtEvtGrp.EVENT_ID = anmEvt.EVENT_ID
+         LEFT JOIN AUDIT_TRAIL adt
+                   ON anmEvt.ANIMAL_EVENT_ID = substring(adt.PRIMARY_KEY_VALUES, length('ANIMAL_EVENT_ID = '))
+                       AND adt.PRIMARY_KEY_VALUES LIKE '%ANIMAL_EVENT_ID%'
+
+WHERE evtEvtGrp.EVENT_GROUP_ID = 56
+--     56 Breeder

nirc_ehr/resources/queries/dbo/q_breeder_delete.sql

@@ -0,0 +1,9 @@
+SELECT substring(adt.PRIMARY_KEY_VALUES, length('ANIMAL_EVENT_ID = ')) AS objectid,
+       CAST(adt.CHANGE_DATETIME AS TIMESTAMP)                            AS modified,
+       adt.REFERENCE
+FROM AUDIT_TRAIL adt
+WHERE adt.PRIMARY_KEY_VALUES LIKE '%ANIMAL_EVENT_ID%' AND
+    (adt.REFERENCE LIKE '%Multiple Sires%' OR
+     adt.REFERENCE LIKE '%Multiple Dams%' OR
+     adt.REFERENCE LIKE '%Mating Event%') AND
+        adt.COLUMN_NAME = 'DELETE'

nirc_ehr/resources/queries/dbo/q_clinremarks.sql

@@ -0,0 +1,20 @@
+SELECT anmEvt.ANIMAL_EVENT_ID                                                    AS objectid,
+       anmEvt.ANIMAL_ID                                                          AS Id,
+       CAST(anmEvt.EVENT_DATETIME AS TIMESTAMP)                                  AS administrationDate,
+       (CASE
+            WHEN anmEvt.STAFF_ID.EMAIL_ADDRESS IS NULL THEN 'unknown'
+            ELSE substr(anmEvt.STAFF_ID.EMAIL_ADDRESS, 1,
+                        instr(anmEvt.STAFF_ID.EMAIL_ADDRESS, '@') - 1) END)      AS performedby,
+       anmEvt.EVENT_ID.EVENT_ID                                                  AS category,
+       anmEvt.DIAGNOSIS                                                          AS hx,
+       anmCmt.TEXT                                                               AS remark,
+       CAST(COALESCE(adt.CHANGE_DATETIME, anmEvt.CREATED_DATETIME) AS TIMESTAMP) AS modified
+FROM ANIMAL_EVENT anmEvt
+         LEFT JOIN ANIMAL_EVENT_COMMENT anmCmt ON anmEvt.ANIMAL_EVENT_ID = anmCmt.ANIMAL_EVENT_ID
+         LEFT JOIN EVENT_EVENT_GROUP evtEvtGrp ON evtEvtGrp.EVENT_ID = anmEvt.EVENT_ID
+         LEFT JOIN AUDIT_TRAIL adt
+                   ON anmEvt.ANIMAL_EVENT_ID = substring(adt.PRIMARY_KEY_VALUES, length('ANIMAL_EVENT_ID = '))
+                       AND adt.PRIMARY_KEY_VALUES LIKE '%ANIMAL_EVENT_ID%'
+
+WHERE evtEvtGrp.EVENT_GROUP_ID = 53
+--     53 Comments

nirc_ehr/resources/queries/dbo/q_clinremarks_delete.sql

@@ -0,0 +1,26 @@
+SELECT substring(adt.PRIMARY_KEY_VALUES, length('ANIMAL_EVENT_ID = ')) AS objectid,
+       CAST(adt.CHANGE_DATETIME AS TIMESTAMP)                            AS modified,
+       adt.REFERENCE
+FROM AUDIT_TRAIL adt
+WHERE adt.PRIMARY_KEY_VALUES LIKE '%ANIMAL_EVENT_ID%' AND
+    (adt.REFERENCE LIKE '%Gross Pathology%' OR
+     adt.REFERENCE LIKE '%Cardiopulmonary Disorder%' OR
+     adt.REFERENCE LIKE '%Dermatological Disorder%' OR
+     adt.REFERENCE LIKE '%Endocrine Disorder%' OR
+     adt.REFERENCE LIKE '%Environmental Exposure%' OR
+     adt.REFERENCE LIKE '%Gastrointestinal Disorder%' OR
+     adt.REFERENCE LIKE '%Hematological Disorder%' OR
+     adt.REFERENCE LIKE '%Iatrogenic%' OR
+     adt.REFERENCE LIKE '%Immune Disorder%' OR
+     adt.REFERENCE LIKE '%Infectious Disease%' OR
+     adt.REFERENCE LIKE '%Metabolic Disorder%' OR
+     adt.REFERENCE LIKE '%Musculoskeletal Disorder%' OR
+     adt.REFERENCE LIKE '%Neoplasia%' OR
+     adt.REFERENCE LIKE '%Neurological Disorder%' OR
+     adt.REFERENCE LIKE '%No Post Mortem exam%' OR
+     adt.REFERENCE LIKE '%Opthalmologic Disorder%' OR
+     adt.REFERENCE LIKE '%Reproductive Disorder%' OR
+     adt.REFERENCE LIKE '%Autolysis%' OR
+     adt.REFERENCE LIKE '%No gross abnormalities%' OR
+     adt.REFERENCE LIKE '%General Comment%') AND
+        adt.COLUMN_NAME = 'DELETE'

nirc_ehr/resources/queries/dbo/q_lookups.sql

@@ -82,12 +82,12 @@ FROM PROTOCOL_CATEGORY
 
 UNION
 
-SELECT 'PROTOCOL_STATE_' || STATE_ID AS objectid,
+SELECT 'PROTOCOL_STATE_' || ID AS objectid,
 'protocol_state' AS set_name,
-STATE_ID AS "value",
-STATE_NAME AS description,
-STATE_NAME AS title,
-STATE_ID AS sort_order,
+ID AS "value",
+DESCRIPTION AS description,
+NAME AS title,
+ID AS sort_order,
 NULL AS date_disabled,
 NULL AS category,
 FROM PROTOCOL_CATEGORY
@@ -105,4 +105,21 @@ WHEN 'Y' THEN NULL
 WHEN 'N' THEN '1/1/1970'
 END AS date_disabled,
 NULL AS category,
-FROM QUESTIONNAIRE
+FROM QUESTIONNAIRE
+
+UNION
+
+SELECT
+'CLINREMARKS_CATEGORY' || EVENT_ID.EVENT_ID as objectid,
+'clinremarks_category' AS set_name,
+EVENT_ID.EVENT_ID AS "value",
+EVENT_ID.DESCRIPTION AS description,
+EVENT_ID.NAME AS title,
+EVENT_ID.EVENT_ID AS sort_order,
+CASE
+    WHEN EVENT_ID.ACTIVE_YN = 'Y' AND EVENT_GROUP_ID.ACTIVE_YN = 'Y' THEN NULL
+    ELSE '1/1/1970'
+    END AS date_disabled,
+NULL AS category,
+FROM EVENT_EVENT_GROUP
+WHERE EVENT_GROUP_ID.EVENT_GROUP_ID = 53

nirc_ehr/resources/queries/study/breeder/.qview.xml

@@ -0,0 +1,12 @@
+<customView xmlns="http://labkey.org/data/xml/queryCustomView">
+    <sorts>
+        <sort column="date" descending="true"/>
+    </sorts>
+    <columns>
+        <column name="Id"/>
+        <column name="date"/>
+        <column name="type"/>
+        <column name="result"/>
+        <column name="remark"/>
+    </columns>
+</customView>

nirc_ehr/resources/queries/study/clinremarks.query.xml

@@ -0,0 +1,18 @@
+<query xmlns="http://labkey.org/data/xml/query">
+    <metadata>
+        <tables xmlns="http://labkey.org/data/xml">
+            <table tableName="clinremarks" tableDbType="TABLE" useColumnOrder="true">
+                <columns>
+                    <column columnName="category">
+                        <columnTitle>Category</columnTitle>
+                        <fk>
+                            <fkDbSchema>ehr_lookups</fkDbSchema>
+                            <fkTable>clinremarks_category</fkTable>
+                            <fkColumnName>value</fkColumnName>
+                        </fk>
+                    </column>
+                </columns>
+            </table>
+        </tables>
+    </metadata>
+</query>

nirc_ehr/resources/queries/study/clinremarks/.qview.xml

@@ -0,0 +1,15 @@
+<customView xmlns="http://labkey.org/data/xml/queryCustomView">
+    <sorts>
+        <sort column="date" descending="true"/>
+    </sorts>
+    <columns>
+        <column name="Id"/>
+        <column name="date"/>
+        <column name="performedby"/>
+        <column name="category"/>
+        <column name="description"/>
+        <column name="remark"/>
+        <column name="history"/>
+        <column name="QCState"/>
+    </columns>
+</customView>

nirc_ehr/resources/queries/study/q_lookup_sets.sql

@@ -58,4 +58,12 @@ SELECT 'questionnaire' AS setname,
        'Questionnaire' AS label,
        'value' AS keyField,
        'title'AS titleColumn,
+       CAST('2021-07-30' AS TIMESTAMP) AS updated_at
+
+UNION
+
+SELECT 'clinremarks_category' AS setname,
+       'Clinical Remarks Category' AS label,
+       'value' AS keyField,
+       'title'AS titleColumn,
        CAST('2021-07-30' AS TIMESTAMP) AS updated_at

nirc_ehr/resources/referenceStudy/datasets/datasets_manifest.xml

@@ -11,6 +11,8 @@
         <dataset name="assignment" id="1002" category="Colony Management" type="Standard"/>
         <dataset name="blood" id="1005" category="Clinical" type="Standard"/>
         <dataset name="birth" id="1007" category="Colony Management" demographicData="true" type="Standard"/>
+        <dataset name="breeder" id="1008" category="Colony Management" type="Standard"/>
+        <dataset name="clinremarks" id="1009" category="Colony Management" type="Standard"/>
         <dataset name="deaths" id="1011" category="Pathology" demographicData="true" type="Standard"/>
         <dataset name="demographics" id="1012" category="Colony Management" demographicData="true" type="Standard"/>
         <dataset name="departure" id="1013" category="Colony Management" type="Standard"/>

nirc_ehr/resources/referenceStudy/datasets/datasets_metadata.xml

@@ -172,6 +172,72 @@
             </column>
         </columns>
     </table>
+    <table tableName="breeder" tableDbType="TABLE">
+        <tableTitle>Breeder</tableTitle>
+        <columns>
+            <column columnName="Id">
+                <datatype>varchar</datatype>
+                <propertyURI>http://cpas.labkey.com/Study#ParticipantId</propertyURI>
+                <importAliases>
+                    <importAlias>ptid</importAlias>
+                </importAliases>
+            </column>
+            <column columnName="date">
+                <datatype>timestamp</datatype>
+                <propertyURI>http://cpas.labkey.com/Study#VisitDate</propertyURI>
+                <conceptURI>http://cpas.labkey.com/Study#VisitDate</conceptURI>
+            </column>
+            <column columnName="objectid">
+                <datatype>varchar</datatype>
+                <propertyURI>urn:ehr.labkey.org/#ObjectId</propertyURI>
+                <isKeyField>true</isKeyField>
+            </column>
+            <column columnName="type">
+                <datatype>varchar</datatype>
+            </column>
+            <column columnName="result">
+                <datatype>varchar</datatype>
+            </column>
+            <column columnName="remark">
+                <datatype>varchar</datatype>
+            </column>
+            <column columnName="attachmentFile">
+                <columnTitle>Attachment File</columnTitle>
+                <datatype>varchar</datatype>
+            </column>
+        </columns>
+    </table>
+    <table tableName="clinremarks" tableDbType="TABLE">
+        <tableTitle>Clinical Remarks</tableTitle>
+        <columns>
+            <column columnName="Id">
+                <datatype>varchar</datatype>
+                <propertyURI>http://cpas.labkey.com/Study#ParticipantId</propertyURI>
+                <importAliases>
+                    <importAlias>ptid</importAlias>
+                </importAliases>
+            </column>
+            <column columnName="date">
+                <datatype>timestamp</datatype>
+                <propertyURI>http://cpas.labkey.com/Study#VisitDate</propertyURI>
+                <conceptURI>http://cpas.labkey.com/Study#VisitDate</conceptURI>
+            </column>
+            <column columnName="objectid">
+                <datatype>varchar</datatype>
+                <propertyURI>urn:ehr.labkey.org/#ObjectId</propertyURI>
+                <isKeyField>true</isKeyField>
+            </column>
+            <column columnName="category">
+                <datatype>varchar</datatype>
+            </column>
+            <column columnName="hx">
+                <datatype>varchar</datatype>
+            </column>
+            <column columnName="remark">
+                <datatype>varchar</datatype>
+            </column>
+        </columns>
+    </table>
     <table tableName="demographics" tableDbType="TABLE">
         <tableTitle>Demographics</tableTitle>
         <columns>

nirc_ehr/resources/schemas/dbscripts/postgresql/nirc_ehr-21.016-21.017.sql

@@ -0,0 +1,4 @@
+SELECT core.executeJavaUpgradeCode('reloadStudy');
+SELECT core.executeJavaUpgradeCode('etl;{NIRC_EHR}/breeder');
+SELECT core.executeJavaUpgradeCode('etl;{NIRC_EHR}/ehr_lookups;truncate');
+SELECT core.executeJavaUpgradeCode('etl;{NIRC_EHR}/clinremarks');

nirc_ehr/resources/views/ehrAdmin.html

@@ -13,7 +13,8 @@
                     {name: 'Manage EHR Study', url: '<%=contextPath%><%=containerPath%>/study-manageStudy.view?'},
                     {name: 'Populate Reports', url: '<%=contextPath%><%=containerPath%>/nirc_ehr-populateData.view?'},
                     {name: 'Set EHR Properties', url: '<%=contextPath%><%=containerPath%>/core-modulePropertyAdmin.view?moduleName=EHR'},
-                    {name: 'Add QC States', url: '<%=contextPath%><%=containerPath%>/ehr-ensureQcStates.view?'}
+                    {name: 'Add QC States', url: '<%=contextPath%><%=containerPath%>/ehr-ensureQcStates.view?'},
+                    {name: 'Prime/Refresh EHR Data Cache', url: '<%=contextPath%>/ehr<%=containerPath%>/primeDataEntryCache.view?'}
                 ]
             },{
                 header: 'Logs',