From 98dec9bc9569964224f03ae6418456ec311ca17d Mon Sep 17 00:00:00 2001 From: John Kirkham Date: Thu, 24 Feb 2022 15:15:49 -0800 Subject: [PATCH 1/5] Rename `C_INT_MAX` for clarity This is simply the largest chunk size that OpenSSL can handle reading and writing before 1.1.1. So clarify the variable name accordingly. --- distributed/comm/tcp.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/distributed/comm/tcp.py b/distributed/comm/tcp.py index f0435002b72..6523915e798 100644 --- a/distributed/comm/tcp.py +++ b/distributed/comm/tcp.py @@ -43,7 +43,7 @@ logger = logging.getLogger(__name__) -C_INT_MAX = 256 ** ctypes.sizeof(ctypes.c_int) // 2 - 1 +OPENSSL_MAX_CHUNKSIZE = 256 ** ctypes.sizeof(ctypes.c_int) // 2 - 1 MAX_BUFFER_SIZE = MEMORY_LIMIT / 2 @@ -214,7 +214,8 @@ async def read(self, deserializers=None): frames = host_array(frames_nbytes) # Workaround for OpenSSL 1.0.2 (can drop with OpenSSL 1.1.1) for i, j in sliding_window( - 2, range(0, frames_nbytes + C_INT_MAX, C_INT_MAX) + 2, + range(0, frames_nbytes + OPENSSL_MAX_CHUNKSIZE, OPENSSL_MAX_CHUNKSIZE), ): chunk = frames[i:j] chunk_nbytes = len(chunk) @@ -352,7 +353,7 @@ class TLS(TCP): """ # Workaround for OpenSSL 1.0.2 (can drop with OpenSSL 1.1.1) - max_shard_size = min(C_INT_MAX, TCP.max_shard_size) + max_shard_size = min(OPENSSL_MAX_CHUNKSIZE, TCP.max_shard_size) def _read_extra(self): TCP._read_extra(self) From e630c6a25570c9366452f3fcc2e623cd1901f8ad Mon Sep 17 00:00:00 2001 From: John Kirkham Date: Thu, 24 Feb 2022 15:15:50 -0800 Subject: [PATCH 2/5] Move comment next to global variable --- distributed/comm/tcp.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/distributed/comm/tcp.py b/distributed/comm/tcp.py index 6523915e798..145b5bcc456 100644 --- a/distributed/comm/tcp.py +++ b/distributed/comm/tcp.py @@ -43,7 +43,9 @@ logger = logging.getLogger(__name__) +# Workaround for OpenSSL 1.0.2 (can drop with OpenSSL 1.1.1) OPENSSL_MAX_CHUNKSIZE = 256 ** ctypes.sizeof(ctypes.c_int) // 2 - 1 + MAX_BUFFER_SIZE = MEMORY_LIMIT / 2 @@ -212,7 +214,6 @@ async def read(self, deserializers=None): (frames_nbytes,) = struct.unpack(fmt, frames_nbytes) frames = host_array(frames_nbytes) - # Workaround for OpenSSL 1.0.2 (can drop with OpenSSL 1.1.1) for i, j in sliding_window( 2, range(0, frames_nbytes + OPENSSL_MAX_CHUNKSIZE, OPENSSL_MAX_CHUNKSIZE), @@ -352,7 +353,6 @@ class TLS(TCP): A TLS-specific version of TCP. """ - # Workaround for OpenSSL 1.0.2 (can drop with OpenSSL 1.1.1) max_shard_size = min(OPENSSL_MAX_CHUNKSIZE, TCP.max_shard_size) def _read_extra(self): From 1fee0ba3640d92c9165c1b259f488fe8d031a50e Mon Sep 17 00:00:00 2001 From: John Kirkham Date: Thu, 24 Feb 2022 15:15:51 -0800 Subject: [PATCH 3/5] Link Python bug report related to this --- distributed/comm/tcp.py | 1 + 1 file changed, 1 insertion(+) diff --git a/distributed/comm/tcp.py b/distributed/comm/tcp.py index 145b5bcc456..afa552826ae 100644 --- a/distributed/comm/tcp.py +++ b/distributed/comm/tcp.py @@ -44,6 +44,7 @@ # Workaround for OpenSSL 1.0.2 (can drop with OpenSSL 1.1.1) +# ref: https://bugs.python.org/issue42853 OPENSSL_MAX_CHUNKSIZE = 256 ** ctypes.sizeof(ctypes.c_int) // 2 - 1 MAX_BUFFER_SIZE = MEMORY_LIMIT / 2 From 2f7eb2fe80724a8f7f72b6fa3fff1f67683a2456 Mon Sep 17 00:00:00 2001 From: John Kirkham Date: Thu, 24 Feb 2022 15:15:52 -0800 Subject: [PATCH 4/5] Use larger OpenSSL chunk size limit w/Python 3.10+ --- distributed/comm/tcp.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/distributed/comm/tcp.py b/distributed/comm/tcp.py index afa552826ae..a735b02240e 100644 --- a/distributed/comm/tcp.py +++ b/distributed/comm/tcp.py @@ -45,7 +45,10 @@ # Workaround for OpenSSL 1.0.2 (can drop with OpenSSL 1.1.1) # ref: https://bugs.python.org/issue42853 -OPENSSL_MAX_CHUNKSIZE = 256 ** ctypes.sizeof(ctypes.c_int) // 2 - 1 +if sys.version_info < (3, 10): + OPENSSL_MAX_CHUNKSIZE = 256 ** ctypes.sizeof(ctypes.c_int) // 2 - 1 +else: + OPENSSL_MAX_CHUNKSIZE = 256 ** ctypes.sizeof(ctypes.c_size_t) - 1 MAX_BUFFER_SIZE = MEMORY_LIMIT / 2 From 3b0622d81a14a751f99503eb8056d462d32d1d85 Mon Sep 17 00:00:00 2001 From: John Kirkham Date: Thu, 24 Feb 2022 15:17:34 -0800 Subject: [PATCH 5/5] Specify Python 3.10+ connection --- distributed/comm/tcp.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/distributed/comm/tcp.py b/distributed/comm/tcp.py index a735b02240e..815586e3c8e 100644 --- a/distributed/comm/tcp.py +++ b/distributed/comm/tcp.py @@ -43,7 +43,8 @@ logger = logging.getLogger(__name__) -# Workaround for OpenSSL 1.0.2 (can drop with OpenSSL 1.1.1) +# Workaround for OpenSSL 1.0.2. +# Can drop with OpenSSL 1.1.1 used by Python 3.10+. # ref: https://bugs.python.org/issue42853 if sys.version_info < (3, 10): OPENSSL_MAX_CHUNKSIZE = 256 ** ctypes.sizeof(ctypes.c_int) // 2 - 1