From 95b0fd208061cc2abedaea4a612063a72b87b12c Mon Sep 17 00:00:00 2001 From: John Kirkham Date: Tue, 21 Jul 2020 14:10:13 -0700 Subject: [PATCH 1/2] Minor whitespace adjustment --- distributed/protocol/utils.py | 1 + 1 file changed, 1 insertion(+) diff --git a/distributed/protocol/utils.py b/distributed/protocol/utils.py index fd50b6ca094..575060149a5 100644 --- a/distributed/protocol/utils.py +++ b/distributed/protocol/utils.py @@ -122,6 +122,7 @@ def unpack_frames(b): """ fmt = "Q" fmt_size = struct.calcsize(fmt) + (n_frames,) = struct.unpack_from(fmt, b) lengths = struct.unpack_from(f"{n_frames}{fmt}", b, fmt_size) From 8a0e4b6a8e4ac4c64ddb7b628af1f5b269b0f3aa Mon Sep 17 00:00:00 2001 From: John Kirkham Date: Tue, 21 Jul 2020 14:10:16 -0700 Subject: [PATCH 2/2] Coerce input to `memoryview` in `unpack_frames` Selecting out each frame from the input causes a copy, which increases memory usage and slows down `unpack_frames`. To fix this, coerce the input to a `memoryview`. This way slices into the `memoryview` only take a view onto the underlying data, which is quite fast and doesn't result in additional memory usage. --- distributed/protocol/utils.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/distributed/protocol/utils.py b/distributed/protocol/utils.py index 575060149a5..8dc75a28c63 100644 --- a/distributed/protocol/utils.py +++ b/distributed/protocol/utils.py @@ -120,6 +120,8 @@ def unpack_frames(b): -------- pack_frames """ + b = memoryview(b) + fmt = "Q" fmt_size = struct.calcsize(fmt)