From ff36eeed60c1468511f0d7a8c4084a119407eb91 Mon Sep 17 00:00:00 2001
From: Kevin Aleman <kevin.aleman@rocket.chat>
Date: Wed, 20 Oct 2021 15:11:36 -0600
Subject: [PATCH 1/3] Error handling on queue processing

---
 app/models/server/raw/OmnichannelQueue.ts     | 15 ++++++++++++++-
 .../server/lib/LivechatEnterprise.js          | 19 +++++++++++++------
 2 files changed, 27 insertions(+), 7 deletions(-)

diff --git a/app/models/server/raw/OmnichannelQueue.ts b/app/models/server/raw/OmnichannelQueue.ts
index 74a2e1d9f8029..672c016fdc2fb 100644
--- a/app/models/server/raw/OmnichannelQueue.ts
+++ b/app/models/server/raw/OmnichannelQueue.ts
@@ -32,12 +32,22 @@ export class OmnichannelQueueRaw extends BaseRaw<IOmnichannelQueueStatus> {
 	}
 
 	async lockQueue() {
+		const date = new Date();
 		const result = await this.col.findOneAndUpdate({
 			_id: UNIQUE_QUEUE_ID,
-			locked: false,
+			$or: [{
+				locked: true,
+				lockedUntil: {
+					$lte: date,
+				},
+			}, {
+				locked: false,
+			}],
 		}, {
 			$set: {
 				locked: true,
+				// apply 5 secs lock lifetime
+				lockedUntil: new Date(date.getTime() + 5000),
 			},
 		}, {
 			sort: {
@@ -55,6 +65,9 @@ export class OmnichannelQueueRaw extends BaseRaw<IOmnichannelQueueStatus> {
 			$set: {
 				locked: false,
 			},
+			$unset: {
+				lockedUntil: 1,
+			},
 		}, {
 			sort: {
 				_id: 1,
diff --git a/ee/app/livechat-enterprise/server/lib/LivechatEnterprise.js b/ee/app/livechat-enterprise/server/lib/LivechatEnterprise.js
index 1da3c3ef1e381..e76d5564a2b49 100644
--- a/ee/app/livechat-enterprise/server/lib/LivechatEnterprise.js
+++ b/ee/app/livechat-enterprise/server/lib/LivechatEnterprise.js
@@ -249,13 +249,20 @@ const queueWorker = {
 
 	async checkQueue(queue) {
 		queueLogger.debug(`Processing items for queue ${ queue || 'Public' }`);
-		if (await OmnichannelQueue.lockQueue()) {
-			await processWaitingQueue(queue);
-			queueLogger.debug(`Queue ${ queue || 'Public' } processed. Unlocking`);
-			await OmnichannelQueue.unlockQueue();
+		try {
+			if (await OmnichannelQueue.lockQueue()) {
+				await processWaitingQueue(queue);
+				queueLogger.debug(`Queue ${ queue || 'Public' } processed. Unlocking`);
+				await OmnichannelQueue.unlockQueue();
+			}
+		} catch (e) {
+			queueLogger.error({
+				msg: `Error processing queue ${ queue || 'public' }`,
+				err: e,
+			});
+		} finally {
+			this.execute();
 		}
-
-		this.execute();
 	},
 };
 

From 7f11b761a1cab1b853b269d5c06559f7a1ecfbed Mon Sep 17 00:00:00 2001
From: Kevin Aleman <kevin.aleman@rocket.chat>
Date: Wed, 20 Oct 2021 15:49:47 -0600
Subject: [PATCH 2/3] add error handling to queue execution and lock timeout

---
 ee/app/livechat-enterprise/server/lib/LivechatEnterprise.js | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/ee/app/livechat-enterprise/server/lib/LivechatEnterprise.js b/ee/app/livechat-enterprise/server/lib/LivechatEnterprise.js
index e76d5564a2b49..0109eceb75fde 100644
--- a/ee/app/livechat-enterprise/server/lib/LivechatEnterprise.js
+++ b/ee/app/livechat-enterprise/server/lib/LivechatEnterprise.js
@@ -254,6 +254,8 @@ const queueWorker = {
 				await processWaitingQueue(queue);
 				queueLogger.debug(`Queue ${ queue || 'Public' } processed. Unlocking`);
 				await OmnichannelQueue.unlockQueue();
+			} else {
+				queueLogger.debug('Queue locked. Waiting');
 			}
 		} catch (e) {
 			queueLogger.error({

From 336f11288b0ca241bf807dcb7bc0e6e6cd7b2874 Mon Sep 17 00:00:00 2001
From: Kevin Aleman <kevin.aleman@rocket.chat>
Date: Wed, 20 Oct 2021 16:12:19 -0600
Subject: [PATCH 3/3] add error handling to queue execution and lock timeout

---
 app/models/server/raw/OmnichannelQueue.ts | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/app/models/server/raw/OmnichannelQueue.ts b/app/models/server/raw/OmnichannelQueue.ts
index 672c016fdc2fb..bca8878d86098 100644
--- a/app/models/server/raw/OmnichannelQueue.ts
+++ b/app/models/server/raw/OmnichannelQueue.ts
@@ -37,8 +37,8 @@ export class OmnichannelQueueRaw extends BaseRaw<IOmnichannelQueueStatus> {
 			_id: UNIQUE_QUEUE_ID,
 			$or: [{
 				locked: true,
-				lockedUntil: {
-					$lte: date,
+				lockedAt: {
+					$lte: new Date(date.getTime() - 5000),
 				},
 			}, {
 				locked: false,
@@ -47,7 +47,7 @@ export class OmnichannelQueueRaw extends BaseRaw<IOmnichannelQueueStatus> {
 			$set: {
 				locked: true,
 				// apply 5 secs lock lifetime
-				lockedUntil: new Date(date.getTime() + 5000),
+				lockedAt: new Date(),
 			},
 		}, {
 			sort: {
@@ -66,7 +66,7 @@ export class OmnichannelQueueRaw extends BaseRaw<IOmnichannelQueueStatus> {
 				locked: false,
 			},
 			$unset: {
-				lockedUntil: 1,
+				lockedAt: 1,
 			},
 		}, {
 			sort: {