From 3a1e75f59c766b8fba173103cd4b5cab58f4ccac Mon Sep 17 00:00:00 2001 From: Alex Waygood Date: Tue, 22 Feb 2022 17:20:20 +0000 Subject: [PATCH 1/7] Add `multiprocessing.reduction` submodule --- pyrightconfig.stricter.json | 1 + stdlib/copyreg.pyi | 3 +- stdlib/multiprocessing/__init__.pyi | 2 +- stdlib/multiprocessing/reduction.pyi | 61 ++++++++++++++++++++++++ tests/stubtest_allowlists/py3_common.txt | 1 - 5 files changed, 65 insertions(+), 3 deletions(-) create mode 100644 stdlib/multiprocessing/reduction.pyi diff --git a/pyrightconfig.stricter.json b/pyrightconfig.stricter.json index caae178514ab..8e55ff80ba04 100644 --- a/pyrightconfig.stricter.json +++ b/pyrightconfig.stricter.json @@ -9,6 +9,7 @@ "**/@python2", "stdlib/distutils/command", "stdlib/lib2to3/refactor.pyi", + "stdlib/multiprocessing/reduction.pyi", "stdlib/sqlite3/dbapi2.pyi", "stdlib/_tkinter.pyi", "stdlib/tkinter", diff --git a/stdlib/copyreg.pyi b/stdlib/copyreg.pyi index 401144a95432..d9d7a7cef935 100644 --- a/stdlib/copyreg.pyi +++ b/stdlib/copyreg.pyi @@ -15,4 +15,5 @@ def add_extension(module: Hashable, name: Hashable, code: SupportsInt) -> None: def remove_extension(module: Hashable, name: Hashable, code: int) -> None: ... def clear_extension_cache() -> None: ... -dispatch_table: dict[type, Callable[[type], str | _Reduce[type]]] # undocumented +_DispatchTableType = dict[type, Callable[[type], str | _Reduce[type]]] # imported by multiprocessing.reduction +dispatch_table: _DispatchTableType # undocumented diff --git a/stdlib/multiprocessing/__init__.pyi b/stdlib/multiprocessing/__init__.pyi index 979176c47a5e..1836017ed428 100644 --- a/stdlib/multiprocessing/__init__.pyi +++ b/stdlib/multiprocessing/__init__.pyi @@ -1,7 +1,7 @@ import sys from collections.abc import Callable, Iterable from logging import Logger -from multiprocessing import connection, context, pool, synchronize +from multiprocessing import connection, context, pool, reduction as reducer, synchronize from multiprocessing.context import ( AuthenticationError as AuthenticationError, BaseContext, diff --git a/stdlib/multiprocessing/reduction.pyi b/stdlib/multiprocessing/reduction.pyi new file mode 100644 index 000000000000..2d897230f123 --- /dev/null +++ b/stdlib/multiprocessing/reduction.pyi @@ -0,0 +1,61 @@ +import pickle +import sys +from abc import ABCMeta +from copyreg import _DispatchTableType +from typing import Any +from typing_extensions import Literal + +if sys.platform == "win32": + __all__ = ["send_handle", "recv_handle", "ForkingPickler", "register", "dump", "DupHandle", "duplicate", "steal_handle"] +else: + __all__ = ["send_handle", "recv_handle", "ForkingPickler", "register", "dump", "DupFd", "sendfds", "recvfds"] + +class ForkingPickler(pickle.Pickler): + dispatch_table: _DispatchTableType + def __init__(self, *args) -> None: ... + @classmethod + def register(cls, type, reduce) -> None: ... + @classmethod + def dumps(cls, obj, protocol: Any | None = ...): ... + loads = pickle.loads + +register = ForkingPickler.register + +def dump(obj, file, protocol: Any | None = ...) -> None: ... + +if sys.platform == "win32": + def duplicate(handle, target_process: Any | None = ..., inheritable: bool = ..., *, source_process: Any | None = ...): ... + def steal_handle(source_pid, handle): ... + def send_handle(conn, handle, destination_pid) -> None: ... + def recv_handle(conn): ... + + class DupHandle: + def __init__(self, handle, access, pid: Any | None = ...) -> None: ... + def detach(self): ... + +else: + if sys.platform == "darwin": + ACKNOWLEDGE: Literal[True] + else: + ACKNOWLEDGE: Literal[False] + + def recvfds(sock, size): ... + def send_handle(conn, handle, destination_pid) -> None: ... + def recv_handle(conn) -> None: ... + def DupFd(fd): ... + +class AbstractReducer(metaclass=ABCMeta): + ForkingPickler = ForkingPickler + register = register + dump = dump + send_handle = send_handle + recv_handle = recv_handle + if sys.platform == "win32": + steal_handle = steal_handle + duplicate = duplicate + DupHandle = DupHandle + else: + sendfds = sendfds + recvfds = recvfds + DupFd = DupFd + def __init__(self, *args) -> None: ... diff --git a/tests/stubtest_allowlists/py3_common.txt b/tests/stubtest_allowlists/py3_common.txt index c37e12f26f9e..de4179347b3f 100644 --- a/tests/stubtest_allowlists/py3_common.txt +++ b/tests/stubtest_allowlists/py3_common.txt @@ -704,7 +704,6 @@ multiprocessing.managers.SyncManager.JoinableQueue multiprocessing.managers.SyncManager.Pool multiprocessing.pool.Pool.Process multiprocessing.pool.ThreadPool.Process -multiprocessing.reducer multiprocessing.synchronize.Semaphore.get_value rlcompleter.Completer.attr_matches rlcompleter.Completer.global_matches From 5939a0f7a54b129339eb4de021f1b3e3f4772fd3 Mon Sep 17 00:00:00 2001 From: Alex Waygood Date: Tue, 22 Feb 2022 17:25:05 +0000 Subject: [PATCH 2/7] Update reduction.pyi --- stdlib/multiprocessing/reduction.pyi | 1 + 1 file changed, 1 insertion(+) diff --git a/stdlib/multiprocessing/reduction.pyi b/stdlib/multiprocessing/reduction.pyi index 2d897230f123..91110ffc6af8 100644 --- a/stdlib/multiprocessing/reduction.pyi +++ b/stdlib/multiprocessing/reduction.pyi @@ -42,6 +42,7 @@ else: def recvfds(sock, size): ... def send_handle(conn, handle, destination_pid) -> None: ... def recv_handle(conn) -> None: ... + def sendfds(sock, fds) -> None: ... def DupFd(fd): ... class AbstractReducer(metaclass=ABCMeta): From 2817a0368bcaa1cf6e47879eb8d266ef77e4cbc0 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Tue, 22 Feb 2022 17:27:26 +0000 Subject: [PATCH 3/7] [pre-commit.ci] auto fixes from pre-commit.com hooks --- stdlib/multiprocessing/reduction.pyi | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/stdlib/multiprocessing/reduction.pyi b/stdlib/multiprocessing/reduction.pyi index 91110ffc6af8..353279500868 100644 --- a/stdlib/multiprocessing/reduction.pyi +++ b/stdlib/multiprocessing/reduction.pyi @@ -42,7 +42,7 @@ else: def recvfds(sock, size): ... def send_handle(conn, handle, destination_pid) -> None: ... def recv_handle(conn) -> None: ... - def sendfds(sock, fds) -> None: ... + def sendfds(sock, fds) -> None: ... def DupFd(fd): ... class AbstractReducer(metaclass=ABCMeta): From cfcd5ae40bdc34898e017f5191272cd9bf49fc0a Mon Sep 17 00:00:00 2001 From: Alex Waygood Date: Tue, 22 Feb 2022 17:37:17 +0000 Subject: [PATCH 4/7] Update reduction.pyi --- stdlib/multiprocessing/reduction.pyi | 40 ++++++++++++++++++++-------- 1 file changed, 29 insertions(+), 11 deletions(-) diff --git a/stdlib/multiprocessing/reduction.pyi b/stdlib/multiprocessing/reduction.pyi index 353279500868..e54f6e421415 100644 --- a/stdlib/multiprocessing/reduction.pyi +++ b/stdlib/multiprocessing/reduction.pyi @@ -45,18 +45,36 @@ else: def sendfds(sock, fds) -> None: ... def DupFd(fd): ... +# These aliases are to work around pyright complaints. +# Pyright doesn't like it when a class object is defined as an alias +# of a global object with the same name. +_ForkingPickler = ForkingPickler +_register = register +_dump = dump +_send_handle = send_handle +_recv_handle = recv_handle + +if sys.platform == "win32": + _steal_handle = steal_handle + _duplicate = duplicate + _DupHandle = DupHandle +else: + _sendfds = sendfds + _recvfds = recvfds + _DupFd = DupFd + class AbstractReducer(metaclass=ABCMeta): - ForkingPickler = ForkingPickler - register = register - dump = dump - send_handle = send_handle - recv_handle = recv_handle + ForkingPickler = _ForkingPickler + register = _register + dump = _dump + send_handle = _send_handle + recv_handle = _recv_handle if sys.platform == "win32": - steal_handle = steal_handle - duplicate = duplicate - DupHandle = DupHandle + steal_handle = _steal_handle + duplicate = _duplicate + DupHandle = _DupHandle else: - sendfds = sendfds - recvfds = recvfds - DupFd = DupFd + sendfds = _sendfds + recvfds = _recvfds + DupFd = _DupFd def __init__(self, *args) -> None: ... From ffd815808ceaef77bd2891daa7b398a25bd395c7 Mon Sep 17 00:00:00 2001 From: Alex Waygood Date: Tue, 22 Feb 2022 17:49:55 +0000 Subject: [PATCH 5/7] Update py3_common.txt --- tests/stubtest_allowlists/py3_common.txt | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/stubtest_allowlists/py3_common.txt b/tests/stubtest_allowlists/py3_common.txt index de4179347b3f..39b09a3dfd0f 100644 --- a/tests/stubtest_allowlists/py3_common.txt +++ b/tests/stubtest_allowlists/py3_common.txt @@ -201,6 +201,8 @@ threading.Lock # A factory function that returns 'most efficient lock'. Marking threading.RLock # Similar to above multiprocessing.dummy.Lock # Similar to above multiprocessing.dummy.RLock # Similar to above +# alias for a class defined elsewhere, mypy infers the variable has type `(*args) -> ForkingPickler` but stubtest infers the runtime type as +multiprocessing.reduction.AbstractReducer.ForkingPickler tkinter.Misc.grid_propagate # The noarg placeholder is a set value list tkinter.Misc.pack_propagate # The noarg placeholder is a set value list tkinter.Tk.eval # from __getattr__ From 21378ed6fa9f26482f00cdc5fa13e2e11c66ef0a Mon Sep 17 00:00:00 2001 From: Alex Waygood Date: Tue, 22 Feb 2022 17:56:39 +0000 Subject: [PATCH 6/7] Allow --- tests/stubtest_allowlists/win32.txt | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tests/stubtest_allowlists/win32.txt b/tests/stubtest_allowlists/win32.txt index f95a753f0f7f..6f25a3ad7bce 100644 --- a/tests/stubtest_allowlists/win32.txt +++ b/tests/stubtest_allowlists/win32.txt @@ -7,6 +7,9 @@ distutils.msvccompiler.HKEYS locale.[A-Z0-9_]+ # Constants that should be moved to _locale and re-exported conditionally locale.nl_langinfo # Function that should be moved to _locale and re-exported conditionally mmap.PAGESIZE +# alias for a class defined elsewhere, +# mypy infers the variable has type `(*args) -> DupHandle` but stubtest infers the runtime type as +multiprocessing.reduction.AbstractReducer.DupHandle msilib.MSI[A-Z_]+ msilib.text.dirname msilib.PID_[A-Z_]+ From 7c325606f59aa1a282d9b7b1cf31d39fe5ca165f Mon Sep 17 00:00:00 2001 From: Alex Waygood Date: Tue, 22 Feb 2022 18:03:00 +0000 Subject: [PATCH 7/7] 3.7 compat --- stdlib/multiprocessing/reduction.pyi | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/stdlib/multiprocessing/reduction.pyi b/stdlib/multiprocessing/reduction.pyi index e54f6e421415..9e7387da64a5 100644 --- a/stdlib/multiprocessing/reduction.pyi +++ b/stdlib/multiprocessing/reduction.pyi @@ -24,7 +24,11 @@ register = ForkingPickler.register def dump(obj, file, protocol: Any | None = ...) -> None: ... if sys.platform == "win32": - def duplicate(handle, target_process: Any | None = ..., inheritable: bool = ..., *, source_process: Any | None = ...): ... + if sys.version_info >= (3, 8): + def duplicate(handle, target_process: Any | None = ..., inheritable: bool = ..., *, source_process: Any | None = ...): ... + else: + def duplicate(handle, target_process: Any | None = ..., inheritable: bool = ...): ... + def steal_handle(source_pid, handle): ... def send_handle(conn, handle, destination_pid) -> None: ... def recv_handle(conn): ...