From 538389eaf60b59891db5d51e49f3f2e59f387ba0 Mon Sep 17 00:00:00 2001 From: Andrew Morgan Date: Fri, 19 Sep 2025 11:54:42 +0100 Subject: [PATCH 1/4] Bump batch size from 50 to 1000 for e2e signature query Increase the batch size from 50 to 1000 when fetching the signatures of E2E keys. This results in fewer queries to the DB, which in one instance, significantly reduced DB pressure on matrix.org. --- synapse/storage/databases/main/end_to_end_keys.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/synapse/storage/databases/main/end_to_end_keys.py b/synapse/storage/databases/main/end_to_end_keys.py index 17ccefe6b5b..ff8ce6bca0b 100644 --- a/synapse/storage/databases/main/end_to_end_keys.py +++ b/synapse/storage/databases/main/end_to_end_keys.py @@ -354,7 +354,7 @@ async def get_e2e_device_keys_and_signatures( if d is not None and d.keys is not None ) - for batch in batch_iter(signature_query, 50): + for batch in batch_iter(signature_query, 1000): cross_sigs_result = ( await self._get_e2e_cross_signing_signatures_for_devices(batch) ) From 090866392d8ab20d4c781c4c65e6b001f3f5b649 Mon Sep 17 00:00:00 2001 From: Andrew Morgan Date: Fri, 19 Sep 2025 12:06:20 +0100 Subject: [PATCH 2/4] newsfile --- changelog.d/18939.misc | 1 + 1 file changed, 1 insertion(+) create mode 100644 changelog.d/18939.misc diff --git a/changelog.d/18939.misc b/changelog.d/18939.misc new file mode 100644 index 00000000000..a1c64104d13 --- /dev/null +++ b/changelog.d/18939.misc @@ -0,0 +1 @@ +Reduce overall number of calls to `_get_e2e_cross_signing_signatures_for_devices` by increasing the limit of devices the query is called with, reducing DB load. From e2d9d3450d38d4d4cd287ea88e5b2ad61b319f5c Mon Sep 17 00:00:00 2001 From: Andrew Morgan Date: Tue, 23 Sep 2025 15:46:39 +0100 Subject: [PATCH 3/4] add comment about batch size origin --- synapse/storage/databases/main/end_to_end_keys.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/synapse/storage/databases/main/end_to_end_keys.py b/synapse/storage/databases/main/end_to_end_keys.py index ff8ce6bca0b..1ec457b0198 100644 --- a/synapse/storage/databases/main/end_to_end_keys.py +++ b/synapse/storage/databases/main/end_to_end_keys.py @@ -354,6 +354,9 @@ async def get_e2e_device_keys_and_signatures( if d is not None and d.keys is not None ) + # 1000 is an arbitrary batch size. It helped performance on a very + # large-scale deployment (matrix.org), but has not been tested against + # any other setup. for batch in batch_iter(signature_query, 1000): cross_sigs_result = ( await self._get_e2e_cross_signing_signatures_for_devices(batch) From f96e65fe6624363ed0d80dd0d48f8bafa9602666 Mon Sep 17 00:00:00 2001 From: Andrew Morgan <1342360+anoadragon453@users.noreply.github.com> Date: Tue, 23 Sep 2025 15:47:02 +0100 Subject: [PATCH 4/4] Update changelog Co-authored-by: Eric Eastwood --- changelog.d/18939.misc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/changelog.d/18939.misc b/changelog.d/18939.misc index a1c64104d13..6cfae2ceafc 100644 --- a/changelog.d/18939.misc +++ b/changelog.d/18939.misc @@ -1 +1 @@ -Reduce overall number of calls to `_get_e2e_cross_signing_signatures_for_devices` by increasing the limit of devices the query is called with, reducing DB load. +Reduce overall number of calls to `_get_e2e_cross_signing_signatures_for_devices` by increasing the batch size of devices the query is called with, reducing DB load.