From 6eb6e91036ce8fa16ee16d2f43406ea3d404ac23 Mon Sep 17 00:00:00 2001 From: labkey-ankurj Date: Mon, 19 Apr 2021 13:20:02 -0700 Subject: [PATCH 1/4] Issue 128 - Fix timeout issue in deaths trigger script --- .../labkey/ehr/utils/TriggerScriptHelper.java | 24 ++++++++++++++++--- 1 file changed, 21 insertions(+), 3 deletions(-) diff --git a/ehr/src/org/labkey/ehr/utils/TriggerScriptHelper.java b/ehr/src/org/labkey/ehr/utils/TriggerScriptHelper.java index 700997975..0ec460d65 100644 --- a/ehr/src/org/labkey/ehr/utils/TriggerScriptHelper.java +++ b/ehr/src/org/labkey/ehr/utils/TriggerScriptHelper.java @@ -32,6 +32,7 @@ import org.labkey.api.data.ContainerManager; import org.labkey.api.data.ConvertHelper; import org.labkey.api.data.DbSchema; +import org.labkey.api.data.DbScope; import org.labkey.api.data.Results; import org.labkey.api.data.ResultsImpl; import org.labkey.api.data.RuntimeSQLException; @@ -90,6 +91,7 @@ import java.sql.SQLException; import java.text.ParseException; import java.text.SimpleDateFormat; +import java.time.LocalDateTime; import java.util.ArrayList; import java.util.Arrays; import java.util.Calendar; @@ -103,6 +105,8 @@ import java.util.Map; import java.util.Set; import java.util.TreeSet; +import java.util.stream.Collectors; +import java.util.stream.IntStream; /** @@ -869,9 +873,23 @@ public void updateDemographicsRecord(List> updatedRows) thro } } - ti.getUpdateService().updateRows(getUser(), getContainer(), newRows, keyRows, null, getExtraContext()); - - EHRDemographicsService.get().getAnimals(getContainer(), ids); + DbScope scope = ti.getSchema().getScope(); + try (DbScope.Transaction transaction = scope.ensureTransaction()) + { + transaction.addCommitTask(() -> + { + try + { + ti.getUpdateService().updateRows(getUser(), getContainer(), newRows, keyRows, null, getExtraContext()); + EHRDemographicsService.get().getAnimals(getContainer(), ids); + } + catch (InvalidKeyException | BatchValidationException | QueryUpdateServiceException | SQLException e) + { + throw new RuntimeException(e); + } + }, DbScope.CommitTaskOption.POSTCOMMIT); + transaction.commit(); + } } public Map getExtraContext() From 70e20f01f0eddb6c281c9ea8a0b7534c5191ddd1 Mon Sep 17 00:00:00 2001 From: labkey-ankurj Date: Mon, 19 Apr 2021 13:23:17 -0700 Subject: [PATCH 2/4] optimize imports --- ehr/src/org/labkey/ehr/utils/TriggerScriptHelper.java | 3 --- 1 file changed, 3 deletions(-) diff --git a/ehr/src/org/labkey/ehr/utils/TriggerScriptHelper.java b/ehr/src/org/labkey/ehr/utils/TriggerScriptHelper.java index 0ec460d65..c7ad73120 100644 --- a/ehr/src/org/labkey/ehr/utils/TriggerScriptHelper.java +++ b/ehr/src/org/labkey/ehr/utils/TriggerScriptHelper.java @@ -91,7 +91,6 @@ import java.sql.SQLException; import java.text.ParseException; import java.text.SimpleDateFormat; -import java.time.LocalDateTime; import java.util.ArrayList; import java.util.Arrays; import java.util.Calendar; @@ -105,8 +104,6 @@ import java.util.Map; import java.util.Set; import java.util.TreeSet; -import java.util.stream.Collectors; -import java.util.stream.IntStream; /** From 483c4c2513351b34142a7cbc624b6631a6cb872d Mon Sep 17 00:00:00 2001 From: labkey-ankurj Date: Mon, 19 Apr 2021 13:35:14 -0700 Subject: [PATCH 3/4] cr feedback - precommit --- ehr/src/org/labkey/ehr/utils/TriggerScriptHelper.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ehr/src/org/labkey/ehr/utils/TriggerScriptHelper.java b/ehr/src/org/labkey/ehr/utils/TriggerScriptHelper.java index c7ad73120..fd329073b 100644 --- a/ehr/src/org/labkey/ehr/utils/TriggerScriptHelper.java +++ b/ehr/src/org/labkey/ehr/utils/TriggerScriptHelper.java @@ -884,7 +884,7 @@ public void updateDemographicsRecord(List> updatedRows) thro { throw new RuntimeException(e); } - }, DbScope.CommitTaskOption.POSTCOMMIT); + }, DbScope.CommitTaskOption.PRECOMMIT); transaction.commit(); } } From 0b5dceb62c2d4e46401c2d84083fc8e93bec953b Mon Sep 17 00:00:00 2001 From: labkey-ankurj Date: Tue, 20 Apr 2021 19:10:36 -0700 Subject: [PATCH 4/4] switch to batching approach --- ehr/resources/queries/study/deaths.js | 14 +++++++++++++ .../labkey/ehr/utils/TriggerScriptHelper.java | 21 +++---------------- 2 files changed, 17 insertions(+), 18 deletions(-) diff --git a/ehr/resources/queries/study/deaths.js b/ehr/resources/queries/study/deaths.js index 9dda9861c..2abcf4d56 100644 --- a/ehr/resources/queries/study/deaths.js +++ b/ehr/resources/queries/study/deaths.js @@ -9,6 +9,8 @@ EHR.Server.Utils = require("ehr/utils").EHR.Server.Utils; var demographicsUpdates = []; var validIds = []; +var totalDemographicsAnimalsUpdated = 0; +var batchSize = 1000; function onInit(event, helper){ helper.setScriptOptions({ @@ -61,6 +63,15 @@ function onUpsert(helper, scriptErrors, row, oldRow){ death: row.date }); } + + if (demographicsUpdates.length >= batchSize) + { + console.log('updating demographics death date for ' + demographicsUpdates.length + " animals"); + helper.getJavaHelper().updateDemographicsRecord(demographicsUpdates); + totalDemographicsAnimalsUpdated += demographicsUpdates.length; + console.log('updated demographics death date for ' + totalDemographicsAnimalsUpdated + " animals"); + demographicsUpdates = []; + } } else { console.log(row.id + " is not a valid id"); @@ -77,9 +88,12 @@ EHR.Server.TriggerManager.registerHandlerForQuery(EHR.Server.TriggerManager.Even function onComplete(event, errors, helper){ + // remaining updates to demographics if (demographicsUpdates.length > 0) { console.log('updating demographics death date for ' + demographicsUpdates.length + " animals"); helper.getJavaHelper().updateDemographicsRecord(demographicsUpdates); + totalDemographicsAnimalsUpdated += demographicsUpdates.length; + console.log('updated demographics death date for ' + totalDemographicsAnimalsUpdated + " animals"); } var deaths = helper.getDeaths(); diff --git a/ehr/src/org/labkey/ehr/utils/TriggerScriptHelper.java b/ehr/src/org/labkey/ehr/utils/TriggerScriptHelper.java index fd329073b..700997975 100644 --- a/ehr/src/org/labkey/ehr/utils/TriggerScriptHelper.java +++ b/ehr/src/org/labkey/ehr/utils/TriggerScriptHelper.java @@ -32,7 +32,6 @@ import org.labkey.api.data.ContainerManager; import org.labkey.api.data.ConvertHelper; import org.labkey.api.data.DbSchema; -import org.labkey.api.data.DbScope; import org.labkey.api.data.Results; import org.labkey.api.data.ResultsImpl; import org.labkey.api.data.RuntimeSQLException; @@ -870,23 +869,9 @@ public void updateDemographicsRecord(List> updatedRows) thro } } - DbScope scope = ti.getSchema().getScope(); - try (DbScope.Transaction transaction = scope.ensureTransaction()) - { - transaction.addCommitTask(() -> - { - try - { - ti.getUpdateService().updateRows(getUser(), getContainer(), newRows, keyRows, null, getExtraContext()); - EHRDemographicsService.get().getAnimals(getContainer(), ids); - } - catch (InvalidKeyException | BatchValidationException | QueryUpdateServiceException | SQLException e) - { - throw new RuntimeException(e); - } - }, DbScope.CommitTaskOption.PRECOMMIT); - transaction.commit(); - } + ti.getUpdateService().updateRows(getUser(), getContainer(), newRows, keyRows, null, getExtraContext()); + + EHRDemographicsService.get().getAnimals(getContainer(), ids); } public Map getExtraContext()