From 0a4eabb88f22fc183f3ded6072f404c8ec495be1 Mon Sep 17 00:00:00 2001 From: Callum Date: Mon, 4 Nov 2019 11:49:59 +0000 Subject: [PATCH 01/34] Add is_reading() method to _UnixReadPipeTransport Unix read pipe is missing method available for other read transports within the aynscio protocol, which checks that the transport read is not paused or the transport is not closing. --- Lib/asyncio/unix_events.py | 3 +++ Lib/test/test_asyncio/test_unix_events.py | 14 ++++++++++++++ 2 files changed, 17 insertions(+) diff --git a/Lib/asyncio/unix_events.py b/Lib/asyncio/unix_events.py index d8f653045aee4c..10946f13e072a7 100644 --- a/Lib/asyncio/unix_events.py +++ b/Lib/asyncio/unix_events.py @@ -522,6 +522,9 @@ def resume_reading(self): if self._loop.get_debug(): logger.debug("%r resumes reading", self) + def is_reading(self): + return not self._paused and not self._closing + def set_protocol(self, protocol): self._protocol = protocol diff --git a/Lib/test/test_asyncio/test_unix_events.py b/Lib/test/test_asyncio/test_unix_events.py index 5487b7afef8326..7037b2091af88d 100644 --- a/Lib/test/test_asyncio/test_unix_events.py +++ b/Lib/test/test_asyncio/test_unix_events.py @@ -740,6 +740,20 @@ def test_resume_reading(self, m_read): tr.resume_reading() self.loop.assert_reader(5, tr._read_ready) + @mock.patch('os.read') + def test_is_reading(self, m_read): + tr = self.read_pipe_transport() + tr._paused = False + tr._closing = False + self.assertTrue(tr.is_reading()) + tr._paused = True + self.assertFalse(tr.is_reading()) + tr._paused = False + tr._closing = True + self.assertFalse(tr.is_reading()) + tr._closing = False + self.assertTrue(tr.is_reading()) + @mock.patch('os.read') def test_close(self, m_read): tr = self.read_pipe_transport() From 6552563b3d5061816720a5a6c7d4ffd6ba35b98b Mon Sep 17 00:00:00 2001 From: Alexandru Ardelean Date: Mon, 4 Nov 2019 16:55:56 +0200 Subject: [PATCH 02/34] bpo-38684: haslib: fix build when Blake2 not enabled in OpenSSL (#17043) --- Misc/NEWS.d/next/Build/2019-11-04-14-30-37.bpo-38684.aed593.rst | 1 + Modules/_hashopenssl.c | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) create mode 100644 Misc/NEWS.d/next/Build/2019-11-04-14-30-37.bpo-38684.aed593.rst diff --git a/Misc/NEWS.d/next/Build/2019-11-04-14-30-37.bpo-38684.aed593.rst b/Misc/NEWS.d/next/Build/2019-11-04-14-30-37.bpo-38684.aed593.rst new file mode 100644 index 00000000000000..c715ff97041275 --- /dev/null +++ b/Misc/NEWS.d/next/Build/2019-11-04-14-30-37.bpo-38684.aed593.rst @@ -0,0 +1 @@ +Fix _hashlib build when Blake2 is disabled, but OpenSSL supports it. diff --git a/Modules/_hashopenssl.c b/Modules/_hashopenssl.c index b147dbe8b3c55d..360e444e7f4aef 100644 --- a/Modules/_hashopenssl.c +++ b/Modules/_hashopenssl.c @@ -42,7 +42,7 @@ #define PY_OPENSSL_HAS_SHAKE 1 #endif -#ifdef NID_blake2b512 +#if defined(NID_blake2b512) && !defined(OPENSSL_NO_BLAKE2) #define PY_OPENSSL_HAS_BLAKE2 1 #endif From f7cd0571619504eec651dff78e8e44eae55595fc Mon Sep 17 00:00:00 2001 From: "blurb-it[bot]" <43283697+blurb-it[bot]@users.noreply.github.com> Date: Mon, 4 Nov 2019 17:20:44 +0000 Subject: [PATCH 03/34] =?UTF-8?q?=F0=9F=93=9C=F0=9F=A4=96=20Added=20by=20b?= =?UTF-8?q?lurb=5Fit.?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../NEWS.d/next/Library/2019-11-04-17-20-43.bpo-38314.zWz6_P.rst | 1 + 1 file changed, 1 insertion(+) create mode 100644 Misc/NEWS.d/next/Library/2019-11-04-17-20-43.bpo-38314.zWz6_P.rst diff --git a/Misc/NEWS.d/next/Library/2019-11-04-17-20-43.bpo-38314.zWz6_P.rst b/Misc/NEWS.d/next/Library/2019-11-04-17-20-43.bpo-38314.zWz6_P.rst new file mode 100644 index 00000000000000..4f957087abdc51 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2019-11-04-17-20-43.bpo-38314.zWz6_P.rst @@ -0,0 +1 @@ +The read pipe on UNIX transports now has access to the `is_reading()` API which is noted in the documentation for asyncio protocols as being generally available on read transports. \ No newline at end of file From f4b1e3d7c64985f5d5b00f6cc9a1c146bbbfd613 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Mon, 4 Nov 2019 19:48:34 +0100 Subject: [PATCH 04/34] bpo-38644: Add Py_EnterRecursiveCall() to the limited API (GH-17046) Provide Py_EnterRecursiveCall() and Py_LeaveRecursiveCall() as regular functions for the limited API. Previously, there were defined as macros, but these macros didn't work with the limited API which cannot access PyThreadState.recursion_depth field. Remove _Py_CheckRecursionLimit from the stable ABI. Add Include/cpython/ceval.h header file. --- Doc/c-api/exceptions.rst | 12 +++-- Doc/whatsnew/3.9.rst | 6 +++ Include/ceval.h | 43 +++------------- Include/cpython/ceval.h | 50 +++++++++++++++++++ Makefile.pre.in | 1 + .../2019-11-04-17-59-46.bpo-38644.euO_RR.rst | 5 ++ PCbuild/pythoncore.vcxproj | 1 + PCbuild/pythoncore.vcxproj.filters | 3 ++ Python/ceval.c | 18 +++++++ 9 files changed, 101 insertions(+), 38 deletions(-) create mode 100644 Include/cpython/ceval.h create mode 100644 Misc/NEWS.d/next/C API/2019-11-04-17-59-46.bpo-38644.euO_RR.rst diff --git a/Doc/c-api/exceptions.rst b/Doc/c-api/exceptions.rst index c7ba74cc8d5874..a042c6eee0a298 100644 --- a/Doc/c-api/exceptions.rst +++ b/Doc/c-api/exceptions.rst @@ -715,15 +715,21 @@ recursion depth automatically). case, a :exc:`RecursionError` is set and a nonzero value is returned. Otherwise, zero is returned. - *where* should be a string such as ``" in instance check"`` to be - concatenated to the :exc:`RecursionError` message caused by the recursion + *where* should be a UTF-8 encoded string such as ``" in instance check"`` to + be concatenated to the :exc:`RecursionError` message caused by the recursion depth limit. -.. c:function:: void Py_LeaveRecursiveCall() + .. versionchanged:: 3.9 + This function is now also available in the limited API. + +.. c:function:: void Py_LeaveRecursiveCall(void) Ends a :c:func:`Py_EnterRecursiveCall`. Must be called once for each *successful* invocation of :c:func:`Py_EnterRecursiveCall`. + .. versionchanged:: 3.9 + This function is now also available in the limited API. + Properly implementing :c:member:`~PyTypeObject.tp_repr` for container types requires special recursion handling. In addition to protecting the stack, :c:member:`~PyTypeObject.tp_repr` also needs to track objects to prevent cycles. The diff --git a/Doc/whatsnew/3.9.rst b/Doc/whatsnew/3.9.rst index 7d7c502459a9b1..3cac9c5eedbc23 100644 --- a/Doc/whatsnew/3.9.rst +++ b/Doc/whatsnew/3.9.rst @@ -197,6 +197,12 @@ Optimizations Build and C API Changes ======================= +* Provide :c:func:`Py_EnterRecursiveCall` and :c:func:`Py_LeaveRecursiveCall` + as regular functions for the limited API. Previously, there were defined as + macros, but these macros didn't work with the limited API which cannot access + ``PyThreadState.recursion_depth`` field. Remove ``_Py_CheckRecursionLimit`` + from the stable ABI. + (Contributed by Victor Stinner in :issue:`38644`.) * Add a new public :c:func:`PyObject_CallNoArgs` function to the C API, which calls a callable Python object without any arguments. It is the most efficient way to call a callable Python object without any argument. diff --git a/Include/ceval.h b/Include/ceval.h index 61db777cc4d5bf..8da779ba97b73e 100644 --- a/Include/ceval.h +++ b/Include/ceval.h @@ -85,41 +85,8 @@ PyAPI_FUNC(int) Py_MakePendingCalls(void); PyAPI_FUNC(void) Py_SetRecursionLimit(int); PyAPI_FUNC(int) Py_GetRecursionLimit(void); -#define Py_EnterRecursiveCall(where) \ - (_Py_MakeRecCheck(PyThreadState_GET()->recursion_depth) && \ - _Py_CheckRecursiveCall(where)) -#define Py_LeaveRecursiveCall() \ - do{ if(_Py_MakeEndRecCheck(PyThreadState_GET()->recursion_depth)) \ - PyThreadState_GET()->overflowed = 0; \ - } while(0) -PyAPI_FUNC(int) _Py_CheckRecursiveCall(const char *where); - -/* Due to the macros in which it's used, _Py_CheckRecursionLimit is in - the stable ABI. It should be removed therefrom when possible. -*/ -PyAPI_DATA(int) _Py_CheckRecursionLimit; - -#ifdef USE_STACKCHECK -/* With USE_STACKCHECK, trigger stack checks in _Py_CheckRecursiveCall() - on every 64th call to Py_EnterRecursiveCall. -*/ -# define _Py_MakeRecCheck(x) \ - (++(x) > _Py_CheckRecursionLimit || \ - ++(PyThreadState_GET()->stackcheck_counter) > 64) -#else -# define _Py_MakeRecCheck(x) (++(x) > _Py_CheckRecursionLimit) -#endif - -/* Compute the "lower-water mark" for a recursion limit. When - * Py_LeaveRecursiveCall() is called with a recursion depth below this mark, - * the overflowed flag is reset to 0. */ -#define _Py_RecursionLimitLowerWaterMark(limit) \ - (((limit) > 200) \ - ? ((limit) - 50) \ - : (3 * ((limit) >> 2))) - -#define _Py_MakeEndRecCheck(x) \ - (--(x) < _Py_RecursionLimitLowerWaterMark(_Py_CheckRecursionLimit)) +PyAPI_FUNC(int) Py_EnterRecursiveCall(const char *where); +PyAPI_FUNC(void) Py_LeaveRecursiveCall(void); #define Py_ALLOW_RECURSION \ do { unsigned char _old = PyThreadState_GET()->recursion_critical;\ @@ -224,6 +191,12 @@ PyAPI_FUNC(int) _PyEval_SliceIndexNotNone(PyObject *, Py_ssize_t *); #define FVS_MASK 0x4 #define FVS_HAVE_SPEC 0x4 +#ifndef Py_LIMITED_API +# define Py_CPYTHON_CEVAL_H +# include "cpython/ceval.h" +# undef Py_CPYTHON_CEVAL_H +#endif + #ifdef __cplusplus } #endif diff --git a/Include/cpython/ceval.h b/Include/cpython/ceval.h new file mode 100644 index 00000000000000..61bbc4f69d531c --- /dev/null +++ b/Include/cpython/ceval.h @@ -0,0 +1,50 @@ +#ifndef Py_CPYTHON_CEVAL_H +# error "this header file must not be included directly" +#endif + +#ifdef __cplusplus +extern "C" { +#endif + +PyAPI_DATA(int) _Py_CheckRecursionLimit; + +#ifdef USE_STACKCHECK +/* With USE_STACKCHECK macro defined, trigger stack checks in + _Py_CheckRecursiveCall() on every 64th call to Py_EnterRecursiveCall. */ +# define _Py_MakeRecCheck(x) \ + (++(x) > _Py_CheckRecursionLimit || \ + ++(PyThreadState_GET()->stackcheck_counter) > 64) +#else +# define _Py_MakeRecCheck(x) (++(x) > _Py_CheckRecursionLimit) +#endif + +PyAPI_FUNC(int) _Py_CheckRecursiveCall(const char *where); + +#define _Py_EnterRecursiveCall_macro(where) \ + (_Py_MakeRecCheck(PyThreadState_GET()->recursion_depth) && \ + _Py_CheckRecursiveCall(where)) + +#define Py_EnterRecursiveCall(where) _Py_EnterRecursiveCall_macro(where) + + +/* Compute the "lower-water mark" for a recursion limit. When + * Py_LeaveRecursiveCall() is called with a recursion depth below this mark, + * the overflowed flag is reset to 0. */ +#define _Py_RecursionLimitLowerWaterMark(limit) \ + (((limit) > 200) \ + ? ((limit) - 50) \ + : (3 * ((limit) >> 2))) + +#define _Py_MakeEndRecCheck(x) \ + (--(x) < _Py_RecursionLimitLowerWaterMark(_Py_CheckRecursionLimit)) + +#define _Py_LeaveRecursiveCall_macro() \ + do{ if(_Py_MakeEndRecCheck(PyThreadState_GET()->recursion_depth)) \ + PyThreadState_GET()->overflowed = 0; \ + } while(0) + +#define Py_LeaveRecursiveCall() _Py_LeaveRecursiveCall_macro() + +#ifdef __cplusplus +} +#endif diff --git a/Makefile.pre.in b/Makefile.pre.in index 1c0958ec974b25..3c607d08ae745d 100644 --- a/Makefile.pre.in +++ b/Makefile.pre.in @@ -1057,6 +1057,7 @@ PYTHON_HEADERS= \ $(srcdir)/Include/Python-ast.h \ \ $(srcdir)/Include/cpython/abstract.h \ + $(srcdir)/Include/cpython/ceval.h \ $(srcdir)/Include/cpython/dictobject.h \ $(srcdir)/Include/cpython/fileobject.h \ $(srcdir)/Include/cpython/import.h \ diff --git a/Misc/NEWS.d/next/C API/2019-11-04-17-59-46.bpo-38644.euO_RR.rst b/Misc/NEWS.d/next/C API/2019-11-04-17-59-46.bpo-38644.euO_RR.rst new file mode 100644 index 00000000000000..b94f505568f41d --- /dev/null +++ b/Misc/NEWS.d/next/C API/2019-11-04-17-59-46.bpo-38644.euO_RR.rst @@ -0,0 +1,5 @@ +Provide :c:func:`Py_EnterRecursiveCall` and :c:func:`Py_LeaveRecursiveCall` +as regular functions for the limited API. Previously, there were defined as +macros, but these macros didn't work with the limited API which cannot access +``PyThreadState.recursion_depth`` field. Remove ``_Py_CheckRecursionLimit`` +from the stable ABI. diff --git a/PCbuild/pythoncore.vcxproj b/PCbuild/pythoncore.vcxproj index 1c055b6a334304..b72474060a3627 100644 --- a/PCbuild/pythoncore.vcxproj +++ b/PCbuild/pythoncore.vcxproj @@ -127,6 +127,7 @@ + diff --git a/PCbuild/pythoncore.vcxproj.filters b/PCbuild/pythoncore.vcxproj.filters index dbff89fbff62ba..80908135550b64 100644 --- a/PCbuild/pythoncore.vcxproj.filters +++ b/PCbuild/pythoncore.vcxproj.filters @@ -84,6 +84,9 @@ Include + + Include + Include diff --git a/Python/ceval.c b/Python/ceval.c index a7d2ea80069a06..881a7dd629bccc 100644 --- a/Python/ceval.c +++ b/Python/ceval.c @@ -5632,3 +5632,21 @@ maybe_dtrace_line(PyFrameObject *frame, } *instr_prev = frame->f_lasti; } + + +/* Implement Py_EnterRecursiveCall() and Py_LeaveRecursiveCall() as functions + for the limited API. */ + +#undef Py_EnterRecursiveCall + +int Py_EnterRecursiveCall(const char *where) +{ + return _Py_EnterRecursiveCall_macro(where); +} + +#undef Py_LeaveRecursiveCall + +void Py_LeaveRecursiveCall(void) +{ + _Py_LeaveRecursiveCall_macro(); +} From 4ef4d510c6822a81367ba5d4eaec759bf26e5297 Mon Sep 17 00:00:00 2001 From: Callum Ward Date: Mon, 4 Nov 2019 20:40:29 +0000 Subject: [PATCH 05/34] Update Misc/NEWS.d/next/Library/2019-11-04-17-20-43.bpo-38314.zWz6_P.rst Co-Authored-By: Kyle Stanley --- .../next/Library/2019-11-04-17-20-43.bpo-38314.zWz6_P.rst | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Misc/NEWS.d/next/Library/2019-11-04-17-20-43.bpo-38314.zWz6_P.rst b/Misc/NEWS.d/next/Library/2019-11-04-17-20-43.bpo-38314.zWz6_P.rst index 4f957087abdc51..9cfdfc7e75af14 100644 --- a/Misc/NEWS.d/next/Library/2019-11-04-17-20-43.bpo-38314.zWz6_P.rst +++ b/Misc/NEWS.d/next/Library/2019-11-04-17-20-43.bpo-38314.zWz6_P.rst @@ -1 +1,2 @@ -The read pipe on UNIX transports now has access to the `is_reading()` API which is noted in the documentation for asyncio protocols as being generally available on read transports. \ No newline at end of file +The read pipe in :mod:`asyncio` UNIX transports now has access to the +`is_reading()` API. From be434dc0380d9f5c7c800de9943cc46d55fd9491 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Tue, 5 Nov 2019 00:51:22 +0100 Subject: [PATCH 06/34] bpo-38644: Pass tstate to Py_EnterRecursiveCall() (GH-16997) * Add _Py_EnterRecursiveCall() and _Py_LeaveRecursiveCall() which require a tstate argument. * Pass tstate to _Py_MakeRecCheck() and _Py_CheckRecursiveCall(). * Convert Py_EnterRecursiveCall() and Py_LeaveRecursiveCall() macros to static inline functions. _PyThreadState_GET() is the most efficient way to get the tstate, and so using it with _Py_EnterRecursiveCall() and _Py_LeaveRecursiveCall() should be a little bit more efficient than using Py_EnterRecursiveCall() and Py_LeaveRecursiveCall() which use the "slower" PyThreadState_GET(). --- Include/cpython/ceval.h | 45 +++++++++++++++-------- Objects/abstract.c | 80 +++++++++++++++++++++++++---------------- Objects/call.c | 42 ++++++++++++---------- Objects/descrobject.c | 34 ++++++++++-------- Objects/methodobject.c | 50 ++++++++++++++------------ Objects/object.c | 70 +++++++++++++++++++++--------------- Python/ceval.c | 14 ++++---- 7 files changed, 200 insertions(+), 135 deletions(-) diff --git a/Include/cpython/ceval.h b/Include/cpython/ceval.h index 61bbc4f69d531c..1e2c4577a78ff7 100644 --- a/Include/cpython/ceval.h +++ b/Include/cpython/ceval.h @@ -11,20 +11,31 @@ PyAPI_DATA(int) _Py_CheckRecursionLimit; #ifdef USE_STACKCHECK /* With USE_STACKCHECK macro defined, trigger stack checks in _Py_CheckRecursiveCall() on every 64th call to Py_EnterRecursiveCall. */ -# define _Py_MakeRecCheck(x) \ - (++(x) > _Py_CheckRecursionLimit || \ - ++(PyThreadState_GET()->stackcheck_counter) > 64) +static inline int _Py_MakeRecCheck(PyThreadState *tstate) { + return (++tstate->recursion_depth > _Py_CheckRecursionLimit + || ++tstate->stackcheck_counter > 64); +} #else -# define _Py_MakeRecCheck(x) (++(x) > _Py_CheckRecursionLimit) +static inline int _Py_MakeRecCheck(PyThreadState *tstate) { + return (++tstate->recursion_depth > _Py_CheckRecursionLimit); +} #endif -PyAPI_FUNC(int) _Py_CheckRecursiveCall(const char *where); +PyAPI_FUNC(int) _Py_CheckRecursiveCall( + PyThreadState *tstate, + const char *where); + +static inline int _Py_EnterRecursiveCall(PyThreadState *tstate, + const char *where) { + return (_Py_MakeRecCheck(tstate) && _Py_CheckRecursiveCall(tstate, where)); +} -#define _Py_EnterRecursiveCall_macro(where) \ - (_Py_MakeRecCheck(PyThreadState_GET()->recursion_depth) && \ - _Py_CheckRecursiveCall(where)) +static inline int _Py_EnterRecursiveCall_inline(const char *where) { + PyThreadState *tstate = PyThreadState_GET(); + return _Py_EnterRecursiveCall(tstate, where); +} -#define Py_EnterRecursiveCall(where) _Py_EnterRecursiveCall_macro(where) +#define Py_EnterRecursiveCall(where) _Py_EnterRecursiveCall_inline(where) /* Compute the "lower-water mark" for a recursion limit. When @@ -38,12 +49,18 @@ PyAPI_FUNC(int) _Py_CheckRecursiveCall(const char *where); #define _Py_MakeEndRecCheck(x) \ (--(x) < _Py_RecursionLimitLowerWaterMark(_Py_CheckRecursionLimit)) -#define _Py_LeaveRecursiveCall_macro() \ - do{ if(_Py_MakeEndRecCheck(PyThreadState_GET()->recursion_depth)) \ - PyThreadState_GET()->overflowed = 0; \ - } while(0) +static inline void _Py_LeaveRecursiveCall(PyThreadState *tstate) { + if (_Py_MakeEndRecCheck(tstate->recursion_depth)) { + tstate->overflowed = 0; + } +} + +static inline void _Py_LeaveRecursiveCall_inline(void) { + PyThreadState *tstate = PyThreadState_GET(); + _Py_LeaveRecursiveCall(tstate); +} -#define Py_LeaveRecursiveCall() _Py_LeaveRecursiveCall_macro() +#define Py_LeaveRecursiveCall() _Py_LeaveRecursiveCall_inline() #ifdef __cplusplus } diff --git a/Objects/abstract.c b/Objects/abstract.c index 3db56fab2c8db5..dc8ba10762de6d 100644 --- a/Objects/abstract.c +++ b/Objects/abstract.c @@ -1,6 +1,7 @@ /* Abstract Object Interface (many thanks to Jim Fulton) */ #include "Python.h" +#include "pycore_pyerrors.h" #include "pycore_pystate.h" #include #include "structmember.h" /* we need the offsetof() macro from there */ @@ -2459,8 +2460,8 @@ recursive_isinstance(PyObject *inst, PyObject *cls) return retval; } -int -PyObject_IsInstance(PyObject *inst, PyObject *cls) +static int +object_isinstance(PyThreadState *tstate, PyObject *inst, PyObject *cls) { _Py_IDENTIFIER(__instancecheck__); PyObject *checker; @@ -2475,34 +2476,31 @@ PyObject_IsInstance(PyObject *inst, PyObject *cls) } if (PyTuple_Check(cls)) { - Py_ssize_t i; - Py_ssize_t n; - int r = 0; - - if (Py_EnterRecursiveCall(" in __instancecheck__")) + if (_Py_EnterRecursiveCall(tstate, " in __instancecheck__")) { return -1; - n = PyTuple_GET_SIZE(cls); - for (i = 0; i < n; ++i) { + } + Py_ssize_t n = PyTuple_GET_SIZE(cls); + int r = 0; + for (Py_ssize_t i = 0; i < n; ++i) { PyObject *item = PyTuple_GET_ITEM(cls, i); - r = PyObject_IsInstance(inst, item); + r = object_isinstance(tstate, inst, item); if (r != 0) /* either found it, or got an error */ break; } - Py_LeaveRecursiveCall(); + _Py_LeaveRecursiveCall(tstate); return r; } checker = _PyObject_LookupSpecial(cls, &PyId___instancecheck__); if (checker != NULL) { - PyObject *res; int ok = -1; - if (Py_EnterRecursiveCall(" in __instancecheck__")) { + if (_Py_EnterRecursiveCall(tstate, " in __instancecheck__")) { Py_DECREF(checker); return ok; } - res = _PyObject_CallOneArg(checker, inst); - Py_LeaveRecursiveCall(); + PyObject *res = _PyObject_CallOneArg(checker, inst); + _Py_LeaveRecursiveCall(tstate); Py_DECREF(checker); if (res != NULL) { ok = PyObject_IsTrue(res); @@ -2510,12 +2508,23 @@ PyObject_IsInstance(PyObject *inst, PyObject *cls) } return ok; } - else if (PyErr_Occurred()) + else if (_PyErr_Occurred(tstate)) { return -1; + } + /* Probably never reached anymore. */ return recursive_isinstance(inst, cls); } + +int +PyObject_IsInstance(PyObject *inst, PyObject *cls) +{ + PyThreadState *tstate = _PyThreadState_GET(); + return object_isinstance(tstate, inst, cls); +} + + static int recursive_issubclass(PyObject *derived, PyObject *cls) { @@ -2534,8 +2543,8 @@ recursive_issubclass(PyObject *derived, PyObject *cls) return abstract_issubclass(derived, cls); } -int -PyObject_IsSubclass(PyObject *derived, PyObject *cls) +static int +object_issubclass(PyThreadState *tstate, PyObject *derived, PyObject *cls) { _Py_IDENTIFIER(__subclasscheck__); PyObject *checker; @@ -2549,34 +2558,32 @@ PyObject_IsSubclass(PyObject *derived, PyObject *cls) } if (PyTuple_Check(cls)) { - Py_ssize_t i; - Py_ssize_t n; - int r = 0; - if (Py_EnterRecursiveCall(" in __subclasscheck__")) + if (_Py_EnterRecursiveCall(tstate, " in __subclasscheck__")) { return -1; - n = PyTuple_GET_SIZE(cls); - for (i = 0; i < n; ++i) { + } + Py_ssize_t n = PyTuple_GET_SIZE(cls); + int r = 0; + for (Py_ssize_t i = 0; i < n; ++i) { PyObject *item = PyTuple_GET_ITEM(cls, i); - r = PyObject_IsSubclass(derived, item); + r = object_issubclass(tstate, derived, item); if (r != 0) /* either found it, or got an error */ break; } - Py_LeaveRecursiveCall(); + _Py_LeaveRecursiveCall(tstate); return r; } checker = _PyObject_LookupSpecial(cls, &PyId___subclasscheck__); if (checker != NULL) { - PyObject *res; int ok = -1; - if (Py_EnterRecursiveCall(" in __subclasscheck__")) { + if (_Py_EnterRecursiveCall(tstate, " in __subclasscheck__")) { Py_DECREF(checker); return ok; } - res = _PyObject_CallOneArg(checker, derived); - Py_LeaveRecursiveCall(); + PyObject *res = _PyObject_CallOneArg(checker, derived); + _Py_LeaveRecursiveCall(tstate); Py_DECREF(checker); if (res != NULL) { ok = PyObject_IsTrue(res); @@ -2584,12 +2591,23 @@ PyObject_IsSubclass(PyObject *derived, PyObject *cls) } return ok; } - else if (PyErr_Occurred()) + else if (_PyErr_Occurred(tstate)) { return -1; + } + /* Probably never reached anymore. */ return recursive_issubclass(derived, cls); } + +int +PyObject_IsSubclass(PyObject *derived, PyObject *cls) +{ + PyThreadState *tstate = _PyThreadState_GET(); + return object_issubclass(tstate, derived, cls); +} + + int _PyObject_RealIsInstance(PyObject *inst, PyObject *cls) { diff --git a/Objects/call.c b/Objects/call.c index a715bcbee4addf..b7588b302fb7d0 100644 --- a/Objects/call.c +++ b/Objects/call.c @@ -1,5 +1,6 @@ #include "Python.h" #include "pycore_object.h" +#include "pycore_pyerrors.h" #include "pycore_pystate.h" #include "pycore_tupleobject.h" #include "frameobject.h" @@ -126,12 +127,15 @@ _PyObject_FastCallDict(PyObject *callable, PyObject *const *args, PyObject * _PyObject_MakeTpCall(PyObject *callable, PyObject *const *args, Py_ssize_t nargs, PyObject *keywords) { + PyThreadState *tstate = _PyThreadState_GET(); + /* Slow path: build a temporary tuple for positional arguments and a * temporary dictionary for keyword arguments (if any) */ ternaryfunc call = Py_TYPE(callable)->tp_call; if (call == NULL) { - PyErr_Format(PyExc_TypeError, "'%.200s' object is not callable", - Py_TYPE(callable)->tp_name); + _PyErr_Format(tstate, PyExc_TypeError, + "'%.200s' object is not callable", + Py_TYPE(callable)->tp_name); return NULL; } @@ -162,10 +166,10 @@ _PyObject_MakeTpCall(PyObject *callable, PyObject *const *args, Py_ssize_t nargs } PyObject *result = NULL; - if (Py_EnterRecursiveCall(" while calling a Python object") == 0) + if (_Py_EnterRecursiveCall(tstate, " while calling a Python object") == 0) { result = call(callable, argstuple, kwdict); - Py_LeaveRecursiveCall(); + _Py_LeaveRecursiveCall(tstate); } Py_DECREF(argstuple); @@ -220,13 +224,14 @@ PyVectorcall_Call(PyObject *callable, PyObject *tuple, PyObject *kwargs) PyObject * PyObject_Call(PyObject *callable, PyObject *args, PyObject *kwargs) { + PyThreadState *tstate = _PyThreadState_GET(); ternaryfunc call; PyObject *result; /* PyObject_Call() must not be called with an exception set, because it can clear it (directly or indirectly) and so the caller loses its exception */ - assert(!PyErr_Occurred()); + assert(!_PyErr_Occurred(tstate)); assert(PyTuple_Check(args)); assert(kwargs == NULL || PyDict_Check(kwargs)); @@ -236,17 +241,19 @@ PyObject_Call(PyObject *callable, PyObject *args, PyObject *kwargs) else { call = callable->ob_type->tp_call; if (call == NULL) { - PyErr_Format(PyExc_TypeError, "'%.200s' object is not callable", - callable->ob_type->tp_name); + _PyErr_Format(tstate, PyExc_TypeError, + "'%.200s' object is not callable", + callable->ob_type->tp_name); return NULL; } - if (Py_EnterRecursiveCall(" while calling a Python object")) + if (_Py_EnterRecursiveCall(tstate, " while calling a Python object")) { return NULL; + } result = (*call)(callable, args, kwargs); - Py_LeaveRecursiveCall(); + _Py_LeaveRecursiveCall(tstate); return _Py_CheckFunctionResult(callable, result, NULL); } @@ -266,30 +273,27 @@ static PyObject* _Py_HOT_FUNCTION function_code_fastcall(PyCodeObject *co, PyObject *const *args, Py_ssize_t nargs, PyObject *globals) { - PyFrameObject *f; + assert(globals != NULL); + PyThreadState *tstate = _PyThreadState_GET(); - PyObject **fastlocals; - Py_ssize_t i; - PyObject *result; + assert(tstate != NULL); - assert(globals != NULL); /* XXX Perhaps we should create a specialized _PyFrame_New_NoTrack() that doesn't take locals, but does take builtins without sanity checking them. */ - assert(tstate != NULL); - f = _PyFrame_New_NoTrack(tstate, co, globals, NULL); + PyFrameObject *f = _PyFrame_New_NoTrack(tstate, co, globals, NULL); if (f == NULL) { return NULL; } - fastlocals = f->f_localsplus; + PyObject **fastlocals = f->f_localsplus; - for (i = 0; i < nargs; i++) { + for (Py_ssize_t i = 0; i < nargs; i++) { Py_INCREF(*args); fastlocals[i] = *args++; } - result = PyEval_EvalFrameEx(f,0); + PyObject *result = PyEval_EvalFrameEx(f, 0); if (Py_REFCNT(f) > 1) { Py_DECREF(f); diff --git a/Objects/descrobject.c b/Objects/descrobject.c index c50fe00ce80bd6..dbab4cd4da2c63 100644 --- a/Objects/descrobject.c +++ b/Objects/descrobject.c @@ -271,9 +271,9 @@ method_check_args(PyObject *func, PyObject *const *args, Py_ssize_t nargs, PyObj } static inline funcptr -method_enter_call(PyObject *func) +method_enter_call(PyThreadState *tstate, PyObject *func) { - if (Py_EnterRecursiveCall(" while calling a Python object")) { + if (_Py_EnterRecursiveCall(tstate, " while calling a Python object")) { return NULL; } return (funcptr)((PyMethodDescrObject *)func)->d_method->ml_meth; @@ -284,6 +284,7 @@ static PyObject * method_vectorcall_VARARGS( PyObject *func, PyObject *const *args, size_t nargsf, PyObject *kwnames) { + PyThreadState *tstate = _PyThreadState_GET(); Py_ssize_t nargs = PyVectorcall_NARGS(nargsf); if (method_check_args(func, args, nargs, kwnames)) { return NULL; @@ -292,14 +293,14 @@ method_vectorcall_VARARGS( if (argstuple == NULL) { return NULL; } - PyCFunction meth = (PyCFunction)method_enter_call(func); + PyCFunction meth = (PyCFunction)method_enter_call(tstate, func); if (meth == NULL) { Py_DECREF(argstuple); return NULL; } PyObject *result = meth(args[0], argstuple); Py_DECREF(argstuple); - Py_LeaveRecursiveCall(); + _Py_LeaveRecursiveCall(tstate); return result; } @@ -307,6 +308,7 @@ static PyObject * method_vectorcall_VARARGS_KEYWORDS( PyObject *func, PyObject *const *args, size_t nargsf, PyObject *kwnames) { + PyThreadState *tstate = _PyThreadState_GET(); Py_ssize_t nargs = PyVectorcall_NARGS(nargsf); if (method_check_args(func, args, nargs, NULL)) { return NULL; @@ -325,12 +327,12 @@ method_vectorcall_VARARGS_KEYWORDS( } } PyCFunctionWithKeywords meth = (PyCFunctionWithKeywords) - method_enter_call(func); + method_enter_call(tstate, func); if (meth == NULL) { goto exit; } result = meth(args[0], argstuple, kwdict); - Py_LeaveRecursiveCall(); + _Py_LeaveRecursiveCall(tstate); exit: Py_DECREF(argstuple); Py_XDECREF(kwdict); @@ -341,17 +343,18 @@ static PyObject * method_vectorcall_FASTCALL( PyObject *func, PyObject *const *args, size_t nargsf, PyObject *kwnames) { + PyThreadState *tstate = _PyThreadState_GET(); Py_ssize_t nargs = PyVectorcall_NARGS(nargsf); if (method_check_args(func, args, nargs, kwnames)) { return NULL; } _PyCFunctionFast meth = (_PyCFunctionFast) - method_enter_call(func); + method_enter_call(tstate, func); if (meth == NULL) { return NULL; } PyObject *result = meth(args[0], args+1, nargs-1); - Py_LeaveRecursiveCall(); + _Py_LeaveRecursiveCall(tstate); return result; } @@ -359,17 +362,18 @@ static PyObject * method_vectorcall_FASTCALL_KEYWORDS( PyObject *func, PyObject *const *args, size_t nargsf, PyObject *kwnames) { + PyThreadState *tstate = _PyThreadState_GET(); Py_ssize_t nargs = PyVectorcall_NARGS(nargsf); if (method_check_args(func, args, nargs, NULL)) { return NULL; } _PyCFunctionFastWithKeywords meth = (_PyCFunctionFastWithKeywords) - method_enter_call(func); + method_enter_call(tstate, func); if (meth == NULL) { return NULL; } PyObject *result = meth(args[0], args+1, nargs-1, kwnames); - Py_LeaveRecursiveCall(); + _Py_LeaveRecursiveCall(tstate); return result; } @@ -377,6 +381,7 @@ static PyObject * method_vectorcall_NOARGS( PyObject *func, PyObject *const *args, size_t nargsf, PyObject *kwnames) { + PyThreadState *tstate = _PyThreadState_GET(); Py_ssize_t nargs = PyVectorcall_NARGS(nargsf); if (method_check_args(func, args, nargs, kwnames)) { return NULL; @@ -386,12 +391,12 @@ method_vectorcall_NOARGS( "%.200s() takes no arguments (%zd given)", get_name(func), nargs-1); return NULL; } - PyCFunction meth = (PyCFunction)method_enter_call(func); + PyCFunction meth = (PyCFunction)method_enter_call(tstate, func); if (meth == NULL) { return NULL; } PyObject *result = meth(args[0], NULL); - Py_LeaveRecursiveCall(); + _Py_LeaveRecursiveCall(tstate); return result; } @@ -399,6 +404,7 @@ static PyObject * method_vectorcall_O( PyObject *func, PyObject *const *args, size_t nargsf, PyObject *kwnames) { + PyThreadState *tstate = _PyThreadState_GET(); Py_ssize_t nargs = PyVectorcall_NARGS(nargsf); if (method_check_args(func, args, nargs, kwnames)) { return NULL; @@ -409,12 +415,12 @@ method_vectorcall_O( get_name(func), nargs-1); return NULL; } - PyCFunction meth = (PyCFunction)method_enter_call(func); + PyCFunction meth = (PyCFunction)method_enter_call(tstate, func); if (meth == NULL) { return NULL; } PyObject *result = meth(args[0], args[1]); - Py_LeaveRecursiveCall(); + _Py_LeaveRecursiveCall(tstate); return result; } diff --git a/Objects/methodobject.c b/Objects/methodobject.c index a5f0c5d3465def..3ce15604b90f1c 100644 --- a/Objects/methodobject.c +++ b/Objects/methodobject.c @@ -3,6 +3,7 @@ #include "Python.h" #include "pycore_object.h" +#include "pycore_pyerrors.h" #include "pycore_pymem.h" #include "pycore_pystate.h" #include "structmember.h" @@ -344,22 +345,22 @@ get_name(PyObject *func) typedef void (*funcptr)(void); static inline int -cfunction_check_kwargs(PyObject *func, PyObject *kwnames) +cfunction_check_kwargs(PyThreadState *tstate, PyObject *func, PyObject *kwnames) { - assert(!PyErr_Occurred()); + assert(!_PyErr_Occurred(tstate)); assert(PyCFunction_Check(func)); if (kwnames && PyTuple_GET_SIZE(kwnames)) { - PyErr_Format(PyExc_TypeError, - "%.200s() takes no keyword arguments", get_name(func)); + _PyErr_Format(tstate, PyExc_TypeError, + "%.200s() takes no keyword arguments", get_name(func)); return -1; } return 0; } static inline funcptr -cfunction_enter_call(PyObject *func) +cfunction_enter_call(PyThreadState *tstate, PyObject *func) { - if (Py_EnterRecursiveCall(" while calling a Python object")) { + if (_Py_EnterRecursiveCall(tstate, " while calling a Python object")) { return NULL; } return (funcptr)PyCFunction_GET_FUNCTION(func); @@ -370,17 +371,18 @@ static PyObject * cfunction_vectorcall_FASTCALL( PyObject *func, PyObject *const *args, size_t nargsf, PyObject *kwnames) { - if (cfunction_check_kwargs(func, kwnames)) { + PyThreadState *tstate = _PyThreadState_GET(); + if (cfunction_check_kwargs(tstate, func, kwnames)) { return NULL; } Py_ssize_t nargs = PyVectorcall_NARGS(nargsf); _PyCFunctionFast meth = (_PyCFunctionFast) - cfunction_enter_call(func); + cfunction_enter_call(tstate, func); if (meth == NULL) { return NULL; } PyObject *result = meth(PyCFunction_GET_SELF(func), args, nargs); - Py_LeaveRecursiveCall(); + _Py_LeaveRecursiveCall(tstate); return result; } @@ -388,14 +390,15 @@ static PyObject * cfunction_vectorcall_FASTCALL_KEYWORDS( PyObject *func, PyObject *const *args, size_t nargsf, PyObject *kwnames) { + PyThreadState *tstate = _PyThreadState_GET(); Py_ssize_t nargs = PyVectorcall_NARGS(nargsf); _PyCFunctionFastWithKeywords meth = (_PyCFunctionFastWithKeywords) - cfunction_enter_call(func); + cfunction_enter_call(tstate, func); if (meth == NULL) { return NULL; } PyObject *result = meth(PyCFunction_GET_SELF(func), args, nargs, kwnames); - Py_LeaveRecursiveCall(); + _Py_LeaveRecursiveCall(tstate); return result; } @@ -403,21 +406,23 @@ static PyObject * cfunction_vectorcall_NOARGS( PyObject *func, PyObject *const *args, size_t nargsf, PyObject *kwnames) { - if (cfunction_check_kwargs(func, kwnames)) { + PyThreadState *tstate = _PyThreadState_GET(); + if (cfunction_check_kwargs(tstate, func, kwnames)) { return NULL; } Py_ssize_t nargs = PyVectorcall_NARGS(nargsf); if (nargs != 0) { - PyErr_Format(PyExc_TypeError, - "%.200s() takes no arguments (%zd given)", get_name(func), nargs); + _PyErr_Format(tstate, PyExc_TypeError, + "%.200s() takes no arguments (%zd given)", + get_name(func), nargs); return NULL; } - PyCFunction meth = (PyCFunction)cfunction_enter_call(func); + PyCFunction meth = (PyCFunction)cfunction_enter_call(tstate, func); if (meth == NULL) { return NULL; } PyObject *result = meth(PyCFunction_GET_SELF(func), NULL); - Py_LeaveRecursiveCall(); + _Py_LeaveRecursiveCall(tstate); return result; } @@ -425,22 +430,23 @@ static PyObject * cfunction_vectorcall_O( PyObject *func, PyObject *const *args, size_t nargsf, PyObject *kwnames) { - if (cfunction_check_kwargs(func, kwnames)) { + PyThreadState *tstate = _PyThreadState_GET(); + if (cfunction_check_kwargs(tstate, func, kwnames)) { return NULL; } Py_ssize_t nargs = PyVectorcall_NARGS(nargsf); if (nargs != 1) { - PyErr_Format(PyExc_TypeError, - "%.200s() takes exactly one argument (%zd given)", - get_name(func), nargs); + _PyErr_Format(tstate, PyExc_TypeError, + "%.200s() takes exactly one argument (%zd given)", + get_name(func), nargs); return NULL; } - PyCFunction meth = (PyCFunction)cfunction_enter_call(func); + PyCFunction meth = (PyCFunction)cfunction_enter_call(tstate, func); if (meth == NULL) { return NULL; } PyObject *result = meth(PyCFunction_GET_SELF(func), args[0]); - Py_LeaveRecursiveCall(); + _Py_LeaveRecursiveCall(tstate); return result; } diff --git a/Objects/object.c b/Objects/object.c index 2c8e823f05ee94..9536d467f5f2a9 100644 --- a/Objects/object.c +++ b/Objects/object.c @@ -2,10 +2,11 @@ /* Generic object operations; and implementation of None */ #include "Python.h" +#include "pycore_context.h" #include "pycore_initconfig.h" #include "pycore_object.h" +#include "pycore_pyerrors.h" #include "pycore_pystate.h" -#include "pycore_context.h" #include "frameobject.h" #include "interpreteridobject.h" @@ -525,31 +526,37 @@ PyObject_Repr(PyObject *v) return PyUnicode_FromFormat("<%s object at %p>", v->ob_type->tp_name, v); + PyThreadState *tstate = _PyThreadState_GET(); #ifdef Py_DEBUG /* PyObject_Repr() must not be called with an exception set, because it can clear it (directly or indirectly) and so the caller loses its exception */ - assert(!PyErr_Occurred()); + assert(!_PyErr_Occurred(tstate)); #endif /* It is possible for a type to have a tp_repr representation that loops infinitely. */ - if (Py_EnterRecursiveCall(" while getting the repr of an object")) + if (_Py_EnterRecursiveCall(tstate, + " while getting the repr of an object")) { return NULL; + } res = (*v->ob_type->tp_repr)(v); - Py_LeaveRecursiveCall(); - if (res == NULL) + _Py_LeaveRecursiveCall(tstate); + + if (res == NULL) { return NULL; + } if (!PyUnicode_Check(res)) { - PyErr_Format(PyExc_TypeError, - "__repr__ returned non-string (type %.200s)", - res->ob_type->tp_name); + _PyErr_Format(tstate, PyExc_TypeError, + "__repr__ returned non-string (type %.200s)", + res->ob_type->tp_name); Py_DECREF(res); return NULL; } #ifndef Py_DEBUG - if (PyUnicode_READY(res) < 0) + if (PyUnicode_READY(res) < 0) { return NULL; + } #endif return res; } @@ -579,31 +586,36 @@ PyObject_Str(PyObject *v) if (Py_TYPE(v)->tp_str == NULL) return PyObject_Repr(v); + PyThreadState *tstate = _PyThreadState_GET(); #ifdef Py_DEBUG /* PyObject_Str() must not be called with an exception set, because it can clear it (directly or indirectly) and so the caller loses its exception */ - assert(!PyErr_Occurred()); + assert(!_PyErr_Occurred(tstate)); #endif /* It is possible for a type to have a tp_str representation that loops infinitely. */ - if (Py_EnterRecursiveCall(" while getting the str of an object")) + if (_Py_EnterRecursiveCall(tstate, " while getting the str of an object")) { return NULL; + } res = (*Py_TYPE(v)->tp_str)(v); - Py_LeaveRecursiveCall(); - if (res == NULL) + _Py_LeaveRecursiveCall(tstate); + + if (res == NULL) { return NULL; + } if (!PyUnicode_Check(res)) { - PyErr_Format(PyExc_TypeError, - "__str__ returned non-string (type %.200s)", - Py_TYPE(res)->tp_name); + _PyErr_Format(tstate, PyExc_TypeError, + "__str__ returned non-string (type %.200s)", + Py_TYPE(res)->tp_name); Py_DECREF(res); return NULL; } #ifndef Py_DEBUG - if (PyUnicode_READY(res) < 0) + if (PyUnicode_READY(res) < 0) { return NULL; + } #endif assert(_PyUnicode_CheckConsistency(res, 1)); return res; @@ -707,7 +719,7 @@ static const char * const opstrings[] = {"<", "<=", "==", "!=", ">", ">="}; /* Perform a rich comparison, raising TypeError when the requested comparison operator is not supported. */ static PyObject * -do_richcompare(PyObject *v, PyObject *w, int op) +do_richcompare(PyThreadState *tstate, PyObject *v, PyObject *w, int op) { richcmpfunc f; PyObject *res; @@ -744,11 +756,11 @@ do_richcompare(PyObject *v, PyObject *w, int op) res = (v != w) ? Py_True : Py_False; break; default: - PyErr_Format(PyExc_TypeError, - "'%s' not supported between instances of '%.100s' and '%.100s'", - opstrings[op], - v->ob_type->tp_name, - w->ob_type->tp_name); + _PyErr_Format(tstate, PyExc_TypeError, + "'%s' not supported between instances of '%.100s' and '%.100s'", + opstrings[op], + v->ob_type->tp_name, + w->ob_type->tp_name); return NULL; } Py_INCREF(res); @@ -761,18 +773,20 @@ do_richcompare(PyObject *v, PyObject *w, int op) PyObject * PyObject_RichCompare(PyObject *v, PyObject *w, int op) { - PyObject *res; + PyThreadState *tstate = _PyThreadState_GET(); assert(Py_LT <= op && op <= Py_GE); if (v == NULL || w == NULL) { - if (!PyErr_Occurred()) + if (!_PyErr_Occurred(tstate)) { PyErr_BadInternalCall(); + } return NULL; } - if (Py_EnterRecursiveCall(" in comparison")) + if (_Py_EnterRecursiveCall(tstate, " in comparison")) { return NULL; - res = do_richcompare(v, w, op); - Py_LeaveRecursiveCall(); + } + PyObject *res = do_richcompare(tstate, v, w, op); + _Py_LeaveRecursiveCall(tstate); return res; } diff --git a/Python/ceval.c b/Python/ceval.c index 881a7dd629bccc..a01fa35dca2d09 100644 --- a/Python/ceval.c +++ b/Python/ceval.c @@ -659,16 +659,15 @@ Py_SetRecursionLimit(int new_limit) _Py_CheckRecursionLimit = ceval->recursion_limit; } -/* the macro Py_EnterRecursiveCall() only calls _Py_CheckRecursiveCall() +/* The function _Py_EnterRecursiveCall() only calls _Py_CheckRecursiveCall() if the recursion_depth reaches _Py_CheckRecursionLimit. If USE_STACKCHECK, the macro decrements _Py_CheckRecursionLimit to guarantee that _Py_CheckRecursiveCall() is regularly called. Without USE_STACKCHECK, there is no need for this. */ int -_Py_CheckRecursiveCall(const char *where) +_Py_CheckRecursiveCall(PyThreadState *tstate, const char *where) { _PyRuntimeState *runtime = &_PyRuntime; - PyThreadState *tstate = _PyRuntimeState_GetThreadState(runtime); int recursion_limit = runtime->ceval.recursion_limit; #ifdef USE_STACKCHECK @@ -1073,8 +1072,9 @@ _PyEval_EvalFrameDefault(PyFrameObject *f, int throwflag) /* Start of code */ /* push frame */ - if (Py_EnterRecursiveCall("")) + if (_Py_EnterRecursiveCall(tstate, "")) { return NULL; + } tstate->frame = f; @@ -3810,7 +3810,7 @@ _PyEval_EvalFrameDefault(PyFrameObject *f, int throwflag) exit_eval_frame: if (PyDTrace_FUNCTION_RETURN_ENABLED()) dtrace_function_return(f); - Py_LeaveRecursiveCall(); + _Py_LeaveRecursiveCall(tstate); f->f_executing = 0; tstate->frame = f->f_back; @@ -5641,12 +5641,12 @@ maybe_dtrace_line(PyFrameObject *frame, int Py_EnterRecursiveCall(const char *where) { - return _Py_EnterRecursiveCall_macro(where); + return _Py_EnterRecursiveCall_inline(where); } #undef Py_LeaveRecursiveCall void Py_LeaveRecursiveCall(void) { - _Py_LeaveRecursiveCall_macro(); + _Py_LeaveRecursiveCall_inline(); } From 17269090940aa20f6079a6b9f27ae319f8cdae14 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Tue, 5 Nov 2019 01:22:12 +0100 Subject: [PATCH 07/34] bpo-38644: Pass tstate to _Py_CheckFunctionResult() (GH-17050) * Add tstate parameter to _Py_CheckFunctionResult() * Add _PyErr_FormatFromCauseTstate() * Replace PyErr_XXX(...) with _PyErr_XXX(state, ...) --- Include/cpython/abstract.h | 12 +++-- Include/internal/pycore_pyerrors.h | 6 +++ Objects/call.c | 81 +++++++++++++++++------------- Objects/methodobject.c | 11 ++-- Objects/typeobject.c | 19 +++---- Python/ceval.c | 2 +- Python/errors.c | 15 ++++++ 7 files changed, 94 insertions(+), 52 deletions(-) diff --git a/Include/cpython/abstract.h b/Include/cpython/abstract.h index 04e4a9e7bd2771..be37e1971a814e 100644 --- a/Include/cpython/abstract.h +++ b/Include/cpython/abstract.h @@ -37,9 +37,11 @@ PyAPI_FUNC(PyObject *) _PyStack_AsDict( 40 bytes on the stack. */ #define _PY_FASTCALL_SMALL_STACK 5 -PyAPI_FUNC(PyObject *) _Py_CheckFunctionResult(PyObject *callable, - PyObject *result, - const char *where); +PyAPI_FUNC(PyObject *) _Py_CheckFunctionResult( + PyThreadState *tstate, + PyObject *callable, + PyObject *result, + const char *where); /* === Vectorcall protocol (PEP 590) ============================= */ @@ -98,13 +100,15 @@ _PyObject_Vectorcall(PyObject *callable, PyObject *const *args, { assert(kwnames == NULL || PyTuple_Check(kwnames)); assert(args != NULL || PyVectorcall_NARGS(nargsf) == 0); + + PyThreadState *tstate = PyThreadState_GET(); vectorcallfunc func = _PyVectorcall_Function(callable); if (func == NULL) { Py_ssize_t nargs = PyVectorcall_NARGS(nargsf); return _PyObject_MakeTpCall(callable, args, nargs, kwnames); } PyObject *res = func(callable, args, nargsf, kwnames); - return _Py_CheckFunctionResult(callable, res, NULL); + return _Py_CheckFunctionResult(tstate, callable, res, NULL); } /* Same as _PyObject_Vectorcall except that keyword arguments are passed as diff --git a/Include/internal/pycore_pyerrors.h b/Include/internal/pycore_pyerrors.h index 2efbf4a62f817c..edbfdfa597e1f0 100644 --- a/Include/internal/pycore_pyerrors.h +++ b/Include/internal/pycore_pyerrors.h @@ -58,6 +58,12 @@ PyAPI_FUNC(void) _PyErr_NormalizeException( PyObject **val, PyObject **tb); +PyAPI_FUNC(PyObject *) _PyErr_FormatFromCauseTstate( + PyThreadState *tstate, + PyObject *exception, + const char *format, + ...); + #ifdef __cplusplus } #endif diff --git a/Objects/call.c b/Objects/call.c index b7588b302fb7d0..0d5c41295c29c8 100644 --- a/Objects/call.c +++ b/Objects/call.c @@ -7,8 +7,9 @@ static PyObject *const * -_PyStack_UnpackDict(PyObject *const *args, Py_ssize_t nargs, PyObject *kwargs, - PyObject **p_kwnames); +_PyStack_UnpackDict(PyThreadState *tstate, + PyObject *const *args, Py_ssize_t nargs, + PyObject *kwargs, PyObject **p_kwnames); static void _PyStack_UnpackDict_Free(PyObject *const *stack, Py_ssize_t nargs, @@ -26,22 +27,23 @@ null_error(void) PyObject* -_Py_CheckFunctionResult(PyObject *callable, PyObject *result, const char *where) +_Py_CheckFunctionResult(PyThreadState *tstate, PyObject *callable, + PyObject *result, const char *where) { - int err_occurred = (PyErr_Occurred() != NULL); + int err_occurred = (_PyErr_Occurred(tstate) != NULL); assert((callable != NULL) ^ (where != NULL)); if (result == NULL) { if (!err_occurred) { if (callable) - PyErr_Format(PyExc_SystemError, - "%R returned NULL without setting an error", - callable); + _PyErr_Format(tstate, PyExc_SystemError, + "%R returned NULL without setting an error", + callable); else - PyErr_Format(PyExc_SystemError, - "%s returned NULL without setting an error", - where); + _PyErr_Format(tstate, PyExc_SystemError, + "%s returned NULL without setting an error", + where); #ifdef Py_DEBUG /* Ensure that the bug is caught in debug mode */ Py_FatalError("a function returned NULL without setting an error"); @@ -54,14 +56,14 @@ _Py_CheckFunctionResult(PyObject *callable, PyObject *result, const char *where) Py_DECREF(result); if (callable) { - _PyErr_FormatFromCause(PyExc_SystemError, - "%R returned a result with an error set", - callable); + _PyErr_FormatFromCauseTstate( + tstate, PyExc_SystemError, + "%R returned a result with an error set", callable); } else { - _PyErr_FormatFromCause(PyExc_SystemError, - "%s returned a result with an error set", - where); + _PyErr_FormatFromCauseTstate( + tstate, PyExc_SystemError, + "%s returned a result with an error set", where); } #ifdef Py_DEBUG /* Ensure that the bug is caught in debug mode */ @@ -88,11 +90,13 @@ PyObject * _PyObject_FastCallDict(PyObject *callable, PyObject *const *args, size_t nargsf, PyObject *kwargs) { + assert(callable != NULL); + + PyThreadState *tstate = _PyThreadState_GET(); /* _PyObject_FastCallDict() must not be called with an exception set, because it can clear it (directly or indirectly) and so the caller loses its exception */ - assert(!PyErr_Occurred()); - assert(callable != NULL); + assert(!_PyErr_Occurred(tstate)); Py_ssize_t nargs = PyVectorcall_NARGS(nargsf); assert(nargs >= 0); @@ -112,7 +116,9 @@ _PyObject_FastCallDict(PyObject *callable, PyObject *const *args, else { PyObject *kwnames; PyObject *const *newargs; - newargs = _PyStack_UnpackDict(args, nargs, kwargs, &kwnames); + newargs = _PyStack_UnpackDict(tstate, + args, nargs, + kwargs, &kwnames); if (newargs == NULL) { return NULL; } @@ -120,7 +126,7 @@ _PyObject_FastCallDict(PyObject *callable, PyObject *const *args, nargs | PY_VECTORCALL_ARGUMENTS_OFFSET, kwnames); _PyStack_UnpackDict_Free(newargs, nargs, kwnames); } - return _Py_CheckFunctionResult(callable, res, NULL); + return _Py_CheckFunctionResult(tstate, callable, res, NULL); } @@ -177,7 +183,7 @@ _PyObject_MakeTpCall(PyObject *callable, PyObject *const *args, Py_ssize_t nargs Py_DECREF(kwdict); } - result = _Py_CheckFunctionResult(callable, result, NULL); + result = _Py_CheckFunctionResult(tstate, callable, result, NULL); return result; } @@ -185,18 +191,22 @@ _PyObject_MakeTpCall(PyObject *callable, PyObject *const *args, Py_ssize_t nargs PyObject * PyVectorcall_Call(PyObject *callable, PyObject *tuple, PyObject *kwargs) { + PyThreadState *tstate = _PyThreadState_GET(); + /* get vectorcallfunc as in _PyVectorcall_Function, but without * the _Py_TPFLAGS_HAVE_VECTORCALL check */ Py_ssize_t offset = Py_TYPE(callable)->tp_vectorcall_offset; if (offset <= 0) { - PyErr_Format(PyExc_TypeError, "'%.200s' object does not support vectorcall", - Py_TYPE(callable)->tp_name); + _PyErr_Format(tstate, PyExc_TypeError, + "'%.200s' object does not support vectorcall", + Py_TYPE(callable)->tp_name); return NULL; } vectorcallfunc func = *(vectorcallfunc *)(((char *)callable) + offset); if (func == NULL) { - PyErr_Format(PyExc_TypeError, "'%.200s' object does not support vectorcall", - Py_TYPE(callable)->tp_name); + _PyErr_Format(tstate, PyExc_TypeError, + "'%.200s' object does not support vectorcall", + Py_TYPE(callable)->tp_name); return NULL; } @@ -210,14 +220,16 @@ PyVectorcall_Call(PyObject *callable, PyObject *tuple, PyObject *kwargs) /* Convert arguments & call */ PyObject *const *args; PyObject *kwnames; - args = _PyStack_UnpackDict(_PyTuple_ITEMS(tuple), nargs, kwargs, &kwnames); + args = _PyStack_UnpackDict(tstate, + _PyTuple_ITEMS(tuple), nargs, + kwargs, &kwnames); if (args == NULL) { return NULL; } PyObject *result = func(callable, args, nargs | PY_VECTORCALL_ARGUMENTS_OFFSET, kwnames); _PyStack_UnpackDict_Free(args, nargs, kwnames); - return _Py_CheckFunctionResult(callable, result, NULL); + return _Py_CheckFunctionResult(tstate, callable, result, NULL); } @@ -255,7 +267,7 @@ PyObject_Call(PyObject *callable, PyObject *args, PyObject *kwargs) _Py_LeaveRecursiveCall(tstate); - return _Py_CheckFunctionResult(callable, result, NULL); + return _Py_CheckFunctionResult(tstate, callable, result, NULL); } } @@ -898,8 +910,9 @@ _PyStack_AsDict(PyObject *const *values, PyObject *kwnames) When done, you must call _PyStack_UnpackDict_Free(stack, nargs, kwnames) */ static PyObject *const * -_PyStack_UnpackDict(PyObject *const *args, Py_ssize_t nargs, PyObject *kwargs, - PyObject **p_kwnames) +_PyStack_UnpackDict(PyThreadState *tstate, + PyObject *const *args, Py_ssize_t nargs, + PyObject *kwargs, PyObject **p_kwnames) { assert(nargs >= 0); assert(kwargs != NULL); @@ -911,14 +924,14 @@ _PyStack_UnpackDict(PyObject *const *args, Py_ssize_t nargs, PyObject *kwargs, * non-negative signed integers, so their difference fits in the type. */ Py_ssize_t maxnargs = PY_SSIZE_T_MAX / sizeof(args[0]) - 1; if (nargs > maxnargs - nkwargs) { - PyErr_NoMemory(); + _PyErr_NoMemory(tstate); return NULL; } /* Add 1 to support PY_VECTORCALL_ARGUMENTS_OFFSET */ PyObject **stack = PyMem_Malloc((1 + nargs + nkwargs) * sizeof(args[0])); if (stack == NULL) { - PyErr_NoMemory(); + _PyErr_NoMemory(tstate); return NULL; } @@ -958,8 +971,8 @@ _PyStack_UnpackDict(PyObject *const *args, Py_ssize_t nargs, PyObject *kwargs, * because it simplifies the deallocation in the failing case. * It happens to also make the loop above slightly more efficient. */ if (!keys_are_strings) { - PyErr_SetString(PyExc_TypeError, - "keywords must be strings"); + _PyErr_SetString(tstate, PyExc_TypeError, + "keywords must be strings"); _PyStack_UnpackDict_Free(stack, nargs, kwnames); return NULL; } diff --git a/Objects/methodobject.c b/Objects/methodobject.c index 3ce15604b90f1c..c780904d7f9aa4 100644 --- a/Objects/methodobject.c +++ b/Objects/methodobject.c @@ -454,9 +454,11 @@ cfunction_vectorcall_O( static PyObject * cfunction_call(PyObject *func, PyObject *args, PyObject *kwargs) { - assert(!PyErr_Occurred()); assert(kwargs == NULL || PyDict_Check(kwargs)); + PyThreadState *tstate = _PyThreadState_GET(); + assert(!_PyErr_Occurred(tstate)); + int flags = PyCFunction_GET_FLAGS(func); if (!(flags & METH_VARARGS)) { /* If this is not a METH_VARARGS function, delegate to vectorcall */ @@ -474,11 +476,12 @@ cfunction_call(PyObject *func, PyObject *args, PyObject *kwargs) } else { if (kwargs != NULL && PyDict_GET_SIZE(kwargs) != 0) { - PyErr_Format(PyExc_TypeError, "%.200s() takes no keyword arguments", - ((PyCFunctionObject*)func)->m_ml->ml_name); + _PyErr_Format(tstate, PyExc_TypeError, + "%.200s() takes no keyword arguments", + ((PyCFunctionObject*)func)->m_ml->ml_name); return NULL; } result = meth(self, args); } - return _Py_CheckFunctionResult(func, result, NULL); + return _Py_CheckFunctionResult(tstate, func, result, NULL); } diff --git a/Objects/typeobject.c b/Objects/typeobject.c index 890246e14a6955..0e1cb7b7aeb369 100644 --- a/Objects/typeobject.c +++ b/Objects/typeobject.c @@ -2,6 +2,7 @@ #include "Python.h" #include "pycore_object.h" +#include "pycore_pyerrors.h" #include "pycore_pystate.h" #include "frameobject.h" #include "structmember.h" @@ -952,12 +953,12 @@ type_repr(PyTypeObject *type) static PyObject * type_call(PyTypeObject *type, PyObject *args, PyObject *kwds) { - PyObject *obj; + PyThreadState *tstate = _PyThreadState_GET(); if (type->tp_new == NULL) { - PyErr_Format(PyExc_TypeError, - "cannot create '%.100s' instances", - type->tp_name); + _PyErr_Format(tstate, PyExc_TypeError, + "cannot create '%.100s' instances", + type->tp_name); return NULL; } @@ -965,11 +966,11 @@ type_call(PyTypeObject *type, PyObject *args, PyObject *kwds) /* type_call() must not be called with an exception set, because it can clear it (directly or indirectly) and so the caller loses its exception */ - assert(!PyErr_Occurred()); + assert(!_PyErr_Occurred(tstate)); #endif - obj = type->tp_new(type, args, kwds); - obj = _Py_CheckFunctionResult((PyObject*)type, obj, NULL); + PyObject *obj = type->tp_new(type, args, kwds); + obj = _Py_CheckFunctionResult(tstate, (PyObject*)type, obj, NULL); if (obj == NULL) return NULL; @@ -990,12 +991,12 @@ type_call(PyTypeObject *type, PyObject *args, PyObject *kwds) if (type->tp_init != NULL) { int res = type->tp_init(obj, args, kwds); if (res < 0) { - assert(PyErr_Occurred()); + assert(_PyErr_Occurred(tstate)); Py_DECREF(obj); obj = NULL; } else { - assert(!PyErr_Occurred()); + assert(!_PyErr_Occurred(tstate)); } } return obj; diff --git a/Python/ceval.c b/Python/ceval.c index a01fa35dca2d09..9019c785080a35 100644 --- a/Python/ceval.c +++ b/Python/ceval.c @@ -3814,7 +3814,7 @@ _PyEval_EvalFrameDefault(PyFrameObject *f, int throwflag) f->f_executing = 0; tstate->frame = f->f_back; - return _Py_CheckFunctionResult(NULL, retval, "PyEval_EvalFrameEx"); + return _Py_CheckFunctionResult(tstate, NULL, retval, "PyEval_EvalFrameEx"); } static void diff --git a/Python/errors.c b/Python/errors.c index b935341636f787..9658afeb9f773f 100644 --- a/Python/errors.c +++ b/Python/errors.c @@ -520,6 +520,21 @@ _PyErr_FormatVFromCause(PyThreadState *tstate, PyObject *exception, return NULL; } +PyObject * +_PyErr_FormatFromCauseTstate(PyThreadState *tstate, PyObject *exception, + const char *format, ...) +{ + va_list vargs; +#ifdef HAVE_STDARG_PROTOTYPES + va_start(vargs, format); +#else + va_start(vargs); +#endif + _PyErr_FormatVFromCause(tstate, exception, format, vargs); + va_end(vargs); + return NULL; +} + PyObject * _PyErr_FormatFromCause(PyObject *exception, const char *format, ...) { From 25fa3ecb98f2c038a422b19c53641fa8e3ef8e52 Mon Sep 17 00:00:00 2001 From: Michael Haas Date: Mon, 4 Nov 2019 22:32:10 -0600 Subject: [PATCH 08/34] Fix a typo in wave module docstring (GH-17009) s/pathing/patching/ --- Lib/wave.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Lib/wave.py b/Lib/wave.py index 100420db9e1620..b7071198e6b841 100644 --- a/Lib/wave.py +++ b/Lib/wave.py @@ -53,7 +53,7 @@ -- set all parameters at once tell() -- return current position in output file writeframesraw(data) - -- write audio frames without pathing up the + -- write audio frames without patching up the file header writeframes(data) -- write audio frames and patch up the file header From 62161ce989d7d4fe2b0e6899a54da20feeddc798 Mon Sep 17 00:00:00 2001 From: Benjamin Peterson Date: Mon, 4 Nov 2019 21:34:14 -0800 Subject: [PATCH 09/34] =?UTF-8?q?closes=20bpo-37633:=20Re=C3=ABxport=20som?= =?UTF-8?q?e=20function=20compatibility=20wrappers=20for=20macros=20in=20`?= =?UTF-8?q?`pythonrun.h``.=20(GH-17056)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../2019-11-04-21-10-47.bpo-37633.oOGVdo.rst | 1 + Python/pythonrun.c | 32 +++++++++---------- 2 files changed, 17 insertions(+), 16 deletions(-) create mode 100644 Misc/NEWS.d/next/C API/2019-11-04-21-10-47.bpo-37633.oOGVdo.rst diff --git a/Misc/NEWS.d/next/C API/2019-11-04-21-10-47.bpo-37633.oOGVdo.rst b/Misc/NEWS.d/next/C API/2019-11-04-21-10-47.bpo-37633.oOGVdo.rst new file mode 100644 index 00000000000000..fdf6abbf1c649b --- /dev/null +++ b/Misc/NEWS.d/next/C API/2019-11-04-21-10-47.bpo-37633.oOGVdo.rst @@ -0,0 +1 @@ +Reëxport some function compatibility wrappers for macros in ``pythonrun.h``. diff --git a/Python/pythonrun.c b/Python/pythonrun.c index 702505893fb991..77a95ceb94bf0c 100644 --- a/Python/pythonrun.c +++ b/Python/pythonrun.c @@ -1648,56 +1648,56 @@ PyOS_CheckStack(void) /* Deprecated C API functions still provided for binary compatibility */ #undef PyParser_SimpleParseFile -node * +PyAPI_FUNC(node *) PyParser_SimpleParseFile(FILE *fp, const char *filename, int start) { return PyParser_SimpleParseFileFlags(fp, filename, start, 0); } #undef PyParser_SimpleParseString -node * +PyAPI_FUNC(node *) PyParser_SimpleParseString(const char *str, int start) { return PyParser_SimpleParseStringFlags(str, start, 0); } #undef PyRun_AnyFile -int +PyAPI_FUNC(int) PyRun_AnyFile(FILE *fp, const char *name) { return PyRun_AnyFileExFlags(fp, name, 0, NULL); } #undef PyRun_AnyFileEx -int +PyAPI_FUNC(int) PyRun_AnyFileEx(FILE *fp, const char *name, int closeit) { return PyRun_AnyFileExFlags(fp, name, closeit, NULL); } #undef PyRun_AnyFileFlags -int +PyAPI_FUNC(int) PyRun_AnyFileFlags(FILE *fp, const char *name, PyCompilerFlags *flags) { return PyRun_AnyFileExFlags(fp, name, 0, flags); } #undef PyRun_File -PyObject * +PyAPI_FUNC(PyObject *) PyRun_File(FILE *fp, const char *p, int s, PyObject *g, PyObject *l) { return PyRun_FileExFlags(fp, p, s, g, l, 0, NULL); } #undef PyRun_FileEx -PyObject * +PyAPI_FUNC(PyObject *) PyRun_FileEx(FILE *fp, const char *p, int s, PyObject *g, PyObject *l, int c) { return PyRun_FileExFlags(fp, p, s, g, l, c, NULL); } #undef PyRun_FileFlags -PyObject * +PyAPI_FUNC(PyObject *) PyRun_FileFlags(FILE *fp, const char *p, int s, PyObject *g, PyObject *l, PyCompilerFlags *flags) { @@ -1705,14 +1705,14 @@ PyRun_FileFlags(FILE *fp, const char *p, int s, PyObject *g, PyObject *l, } #undef PyRun_SimpleFile -int +PyAPI_FUNC(int) PyRun_SimpleFile(FILE *f, const char *p) { return PyRun_SimpleFileExFlags(f, p, 0, NULL); } #undef PyRun_SimpleFileEx -int +PyAPI_FUNC(int) PyRun_SimpleFileEx(FILE *f, const char *p, int c) { return PyRun_SimpleFileExFlags(f, p, c, NULL); @@ -1720,28 +1720,28 @@ PyRun_SimpleFileEx(FILE *f, const char *p, int c) #undef PyRun_String -PyObject * +PyAPI_FUNC(PyObject *) PyRun_String(const char *str, int s, PyObject *g, PyObject *l) { return PyRun_StringFlags(str, s, g, l, NULL); } #undef PyRun_SimpleString -int +PyAPI_FUNC(int) PyRun_SimpleString(const char *s) { return PyRun_SimpleStringFlags(s, NULL); } #undef Py_CompileString -PyObject * +PyAPI_FUNC(PyObject *) Py_CompileString(const char *str, const char *p, int s) { return Py_CompileStringExFlags(str, p, s, NULL, -1); } #undef Py_CompileStringFlags -PyObject * +PyAPI_FUNC(PyObject *) Py_CompileStringFlags(const char *str, const char *p, int s, PyCompilerFlags *flags) { @@ -1749,14 +1749,14 @@ Py_CompileStringFlags(const char *str, const char *p, int s, } #undef PyRun_InteractiveOne -int +PyAPI_FUNC(int) PyRun_InteractiveOne(FILE *f, const char *p) { return PyRun_InteractiveOneFlags(f, p, NULL); } #undef PyRun_InteractiveLoop -int +PyAPI_FUNC(int) PyRun_InteractiveLoop(FILE *f, const char *p) { return PyRun_InteractiveLoopFlags(f, p, NULL); From fbbfcce2d6c5ab16324b36572966e9605e7dc192 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Tue, 5 Nov 2019 11:44:28 +0100 Subject: [PATCH 10/34] _json.c: use Py_UNUSED() macro (GH-17053) Remove UNUSED macro: use Py_UNUSED() macro instead. --- Modules/_json.c | 14 ++++---------- 1 file changed, 4 insertions(+), 10 deletions(-) diff --git a/Modules/_json.c b/Modules/_json.c index 54ac605fd7ef42..439414fd59e621 100644 --- a/Modules/_json.c +++ b/Modules/_json.c @@ -12,12 +12,6 @@ #include "structmember.h" #include "pycore_accu.h" -#ifdef __GNUC__ -#define UNUSED __attribute__((__unused__)) -#else -#define UNUSED -#endif - #define PyScanner_Check(op) PyObject_TypeCheck(op, &PyScannerType) #define PyScanner_CheckExact(op) (Py_TYPE(op) == &PyScannerType) #define PyEncoder_Check(op) PyObject_TypeCheck(op, &PyEncoderType) @@ -78,7 +72,7 @@ static PyMemberDef encoder_members[] = { static PyObject * ascii_escape_unicode(PyObject *pystr); static PyObject * -py_encode_basestring_ascii(PyObject* self UNUSED, PyObject *pystr); +py_encode_basestring_ascii(PyObject* Py_UNUSED(self), PyObject *pystr); void init_json(void); static PyObject * scan_once_unicode(PyScannerObject *s, PyObject *pystr, Py_ssize_t idx, Py_ssize_t *next_idx_ptr); @@ -562,7 +556,7 @@ PyDoc_STRVAR(pydoc_scanstring, ); static PyObject * -py_scanstring(PyObject* self UNUSED, PyObject *args) +py_scanstring(PyObject* Py_UNUSED(self), PyObject *args) { PyObject *pystr; PyObject *rval; @@ -591,7 +585,7 @@ PyDoc_STRVAR(pydoc_encode_basestring_ascii, ); static PyObject * -py_encode_basestring_ascii(PyObject* self UNUSED, PyObject *pystr) +py_encode_basestring_ascii(PyObject* Py_UNUSED(self), PyObject *pystr) { PyObject *rval; /* Return an ASCII-only JSON representation of a Python string */ @@ -616,7 +610,7 @@ PyDoc_STRVAR(pydoc_encode_basestring, ); static PyObject * -py_encode_basestring(PyObject* self UNUSED, PyObject *pystr) +py_encode_basestring(PyObject* Py_UNUSED(self), PyObject *pystr) { PyObject *rval; /* Return a JSON representation of a Python string */ From 5e01a6542a1beb552a17e16b71dc0ba9fc6adcfb Mon Sep 17 00:00:00 2001 From: "Jules Lasne (jlasne)" Date: Tue, 5 Nov 2019 14:20:38 +0100 Subject: [PATCH 11/34] Update interpreter.rst (GH-17059) Fixed what seemed to be a weird phrasing. --- Doc/tutorial/interpreter.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Doc/tutorial/interpreter.rst b/Doc/tutorial/interpreter.rst index ffab8fa4e0d84d..b78d2960ac5432 100644 --- a/Doc/tutorial/interpreter.rst +++ b/Doc/tutorial/interpreter.rst @@ -23,7 +23,7 @@ is an installation option, other places are possible; check with your local Python guru or system administrator. (E.g., :file:`/usr/local/python` is a popular alternative location.) -On Windows machines where you have installed from the :ref:`Microsoft Store +On Windows machines where you have installed Python from the :ref:`Microsoft Store `, the :file:`python3.9` command will be available. If you have the :ref:`py.exe launcher ` installed, you can use the :file:`py` command. See :ref:`setting-envvars` for other ways to launch Python. From b3966639d28313809774ca3859a347b9007be8d2 Mon Sep 17 00:00:00 2001 From: Eddie Elizondo Date: Tue, 5 Nov 2019 07:16:14 -0800 Subject: [PATCH 12/34] bpo-35381 Remove all static state from posixmodule (GH-15892) After #9665, this moves the remaining types in posixmodule to be heap-allocated to make it compatible with PEP384 as well as modifying all the type accessors to fully make the type opaque. The original PR that got messed up a rebase: https://github.com/python/cpython/pull/10854. All the issues in that commit have now been addressed since https://github.com/python/cpython/pull/11661 got committed. This change also removes any state from the data segment and onto the module state itself. https://bugs.python.org/issue35381 Automerge-Triggered-By: @encukou --- Lib/test/test_os.py | 30 ++ .../2019-01-18-17-05-26.bpo-35381.9CbeW3.rst | 2 + Modules/clinic/posixmodule.c.h | 4 +- Modules/posixmodule.c | 495 ++++++++++-------- 4 files changed, 305 insertions(+), 226 deletions(-) create mode 100644 Misc/NEWS.d/next/C API/2019-01-18-17-05-26.bpo-35381.9CbeW3.rst diff --git a/Lib/test/test_os.py b/Lib/test/test_os.py index d2bd9c2f232f1e..bf40cb1e8fa7da 100644 --- a/Lib/test/test_os.py +++ b/Lib/test/test_os.py @@ -3638,6 +3638,24 @@ def test_os_all(self): self.assertIn('walk', os.__all__) +class TestDirEntry(unittest.TestCase): + def setUp(self): + self.path = os.path.realpath(support.TESTFN) + self.addCleanup(support.rmtree, self.path) + os.mkdir(self.path) + + def test_uninstantiable(self): + self.assertRaises(TypeError, os.DirEntry) + + def test_unpickable(self): + filename = create_file(os.path.join(self.path, "file.txt"), b'python') + entry = [entry for entry in os.scandir(self.path)].pop() + self.assertIsInstance(entry, os.DirEntry) + self.assertEqual(entry.name, "file.txt") + import pickle + self.assertRaises(TypeError, pickle.dumps, entry, filename) + + class TestScandir(unittest.TestCase): check_no_resource_warning = support.check_no_resource_warning @@ -3672,6 +3690,18 @@ def assert_stat_equal(self, stat1, stat2, skip_fields): else: self.assertEqual(stat1, stat2) + def test_uninstantiable(self): + scandir_iter = os.scandir(self.path) + self.assertRaises(TypeError, type(scandir_iter)) + scandir_iter.close() + + def test_unpickable(self): + filename = self.create_file("file.txt") + scandir_iter = os.scandir(self.path) + import pickle + self.assertRaises(TypeError, pickle.dumps, scandir_iter, filename) + scandir_iter.close() + def check_entry(self, entry, name, is_dir, is_file, is_symlink): self.assertIsInstance(entry, os.DirEntry) self.assertEqual(entry.name, name) diff --git a/Misc/NEWS.d/next/C API/2019-01-18-17-05-26.bpo-35381.9CbeW3.rst b/Misc/NEWS.d/next/C API/2019-01-18-17-05-26.bpo-35381.9CbeW3.rst new file mode 100644 index 00000000000000..a7efce468c6d81 --- /dev/null +++ b/Misc/NEWS.d/next/C API/2019-01-18-17-05-26.bpo-35381.9CbeW3.rst @@ -0,0 +1,2 @@ +Convert posixmodule.c statically allocated types ``DirEntryType`` and +``ScandirIteratorType`` to heap-allocated types. diff --git a/Modules/clinic/posixmodule.c.h b/Modules/clinic/posixmodule.c.h index 561cbb0ca82ab3..3dada674fbab0d 100644 --- a/Modules/clinic/posixmodule.c.h +++ b/Modules/clinic/posixmodule.c.h @@ -2838,7 +2838,7 @@ PyDoc_STRVAR(os_sched_param__doc__, "sched_param(sched_priority)\n" "--\n" "\n" -"Current has only one field: sched_priority\");\n" +"Currently has only one field: sched_priority\n" "\n" " sched_priority\n" " A scheduling parameter."); @@ -8731,4 +8731,4 @@ os__remove_dll_directory(PyObject *module, PyObject *const *args, Py_ssize_t nar #ifndef OS__REMOVE_DLL_DIRECTORY_METHODDEF #define OS__REMOVE_DLL_DIRECTORY_METHODDEF #endif /* !defined(OS__REMOVE_DLL_DIRECTORY_METHODDEF) */ -/*[clinic end generated code: output=fe7897441fed5402 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=c6e67d475eef00c4 input=a9049054013a1b77]*/ diff --git a/Modules/posixmodule.c b/Modules/posixmodule.c index dcd90d3a5152d7..6d837c6552f490 100644 --- a/Modules/posixmodule.c +++ b/Modules/posixmodule.c @@ -217,6 +217,7 @@ corresponding Unix manual entries for more information on calls."); #endif /* _MSC_VER */ #endif /* ! __WATCOMC__ || __QNX__ */ +_Py_IDENTIFIER(__fspath__); /*[clinic input] # one of the few times we lie about this name! @@ -537,7 +538,7 @@ _Py_Uid_Converter(PyObject *obj, void *p) if (index == NULL) { PyErr_Format(PyExc_TypeError, "uid should be integer, not %.200s", - Py_TYPE(obj)->tp_name); + _PyType_Name(Py_TYPE(obj))); return 0; } @@ -643,7 +644,7 @@ _Py_Gid_Converter(PyObject *obj, void *p) if (index == NULL) { PyErr_Format(PyExc_TypeError, "gid should be integer, not %.200s", - Py_TYPE(obj)->tp_name); + _PyType_Name(Py_TYPE(obj))); return 0; } @@ -810,11 +811,37 @@ dir_fd_converter(PyObject *o, void *p) else { PyErr_Format(PyExc_TypeError, "argument should be integer or None, not %.200s", - Py_TYPE(o)->tp_name); + _PyType_Name(Py_TYPE(o))); return 0; } } +typedef struct { + PyObject *billion; + PyObject *posix_putenv_garbage; + PyObject *DirEntryType; + PyObject *ScandirIteratorType; +#if defined(HAVE_SCHED_SETPARAM) || defined(HAVE_SCHED_SETSCHEDULER) || defined(POSIX_SPAWN_SETSCHEDULER) || defined(POSIX_SPAWN_SETSCHEDPARAM) + PyObject *SchedParamType; +#endif + PyObject *StatResultType; + PyObject *StatVFSResultType; + PyObject *TerminalSizeType; + PyObject *TimesResultType; + PyObject *UnameResultType; +#if defined(HAVE_WAITID) && !defined(__APPLE__) + PyObject *WaitidResultType; +#endif +#if defined(HAVE_WAIT3) || defined(HAVE_WAIT4) + PyObject *struct_rusage; +#endif + PyObject *st_mode; +} _posixstate; + +static struct PyModuleDef posixmodule; + +#define _posixstate(o) ((_posixstate *)PyModule_GetState(o)) +#define _posixstate_global ((_posixstate *)PyModule_GetState(PyState_FindModule(&posixmodule))) /* * A PyArg_ParseTuple "converter" function @@ -984,7 +1011,6 @@ path_converter(PyObject *o, void *p) if (!is_index && !is_buffer && !is_unicode && !is_bytes) { /* Inline PyOS_FSPath() for better error messages. */ - _Py_IDENTIFIER(__fspath__); PyObject *func, *res; func = _PyObject_LookupSpecial(o, &PyId___fspath__); @@ -1005,8 +1031,8 @@ path_converter(PyObject *o, void *p) else { PyErr_Format(PyExc_TypeError, "expected %.200s.__fspath__() to return str or bytes, " - "not %.200s", Py_TYPE(o)->tp_name, - Py_TYPE(res)->tp_name); + "not %.200s", _PyType_Name(Py_TYPE(o)), + _PyType_Name(Py_TYPE(res))); Py_DECREF(res); goto error_exit; } @@ -1058,7 +1084,7 @@ path_converter(PyObject *o, void *p) path->allow_fd ? "string, bytes, os.PathLike or integer" : path->nullable ? "string, bytes, os.PathLike or None" : "string, bytes or os.PathLike", - Py_TYPE(o)->tp_name)) { + _PyType_Name(Py_TYPE(o)))) { goto error_exit; } bytes = PyBytes_FromObject(o); @@ -1089,7 +1115,7 @@ path_converter(PyObject *o, void *p) path->allow_fd ? "string, bytes, os.PathLike or integer" : path->nullable ? "string, bytes, os.PathLike or None" : "string, bytes or os.PathLike", - Py_TYPE(o)->tp_name); + _PyType_Name(Py_TYPE(o))); goto error_exit; } @@ -2047,14 +2073,6 @@ static PyStructSequence_Desc waitid_result_desc = { waitid_result_fields, 5 }; -static PyTypeObject* WaitidResultType; -#endif - -static int initialized; -static PyTypeObject* StatResultType; -static PyTypeObject* StatVFSResultType; -#if defined(HAVE_SCHED_SETPARAM) || defined(HAVE_SCHED_SETSCHEDULER) || defined(POSIX_SPAWN_SETSCHEDULER) || defined(POSIX_SPAWN_SETSCHEDPARAM) -static PyTypeObject* SchedParamType; #endif static newfunc structseq_new; @@ -2080,8 +2098,61 @@ statresult_new(PyTypeObject *type, PyObject *args, PyObject *kwds) return (PyObject*)result; } +static int +_posix_clear(PyObject *module) +{ + Py_CLEAR(_posixstate(module)->billion); + Py_CLEAR(_posixstate(module)->posix_putenv_garbage); + Py_CLEAR(_posixstate(module)->DirEntryType); + Py_CLEAR(_posixstate(module)->ScandirIteratorType); +#if defined(HAVE_SCHED_SETPARAM) || defined(HAVE_SCHED_SETSCHEDULER) || defined(POSIX_SPAWN_SETSCHEDULER) || defined(POSIX_SPAWN_SETSCHEDPARAM) + Py_CLEAR(_posixstate(module)->SchedParamType); +#endif + Py_CLEAR(_posixstate(module)->StatResultType); + Py_CLEAR(_posixstate(module)->StatVFSResultType); + Py_CLEAR(_posixstate(module)->TerminalSizeType); + Py_CLEAR(_posixstate(module)->TimesResultType); + Py_CLEAR(_posixstate(module)->UnameResultType); +#if defined(HAVE_WAITID) && !defined(__APPLE__) + Py_CLEAR(_posixstate(module)->WaitidResultType); +#endif +#if defined(HAVE_WAIT3) || defined(HAVE_WAIT4) + Py_CLEAR(_posixstate(module)->struct_rusage); +#endif + Py_CLEAR(_posixstate(module)->st_mode); + return 0; +} -static PyObject *billion = NULL; +static int +_posix_traverse(PyObject *module, visitproc visit, void *arg) +{ + Py_VISIT(_posixstate(module)->billion); + Py_VISIT(_posixstate(module)->posix_putenv_garbage); + Py_VISIT(_posixstate(module)->DirEntryType); + Py_VISIT(_posixstate(module)->ScandirIteratorType); +#if defined(HAVE_SCHED_SETPARAM) || defined(HAVE_SCHED_SETSCHEDULER) || defined(POSIX_SPAWN_SETSCHEDULER) || defined(POSIX_SPAWN_SETSCHEDPARAM) + Py_VISIT(_posixstate(module)->SchedParamType); +#endif + Py_VISIT(_posixstate(module)->StatResultType); + Py_VISIT(_posixstate(module)->StatVFSResultType); + Py_VISIT(_posixstate(module)->TerminalSizeType); + Py_VISIT(_posixstate(module)->TimesResultType); + Py_VISIT(_posixstate(module)->UnameResultType); +#if defined(HAVE_WAITID) && !defined(__APPLE__) + Py_VISIT(_posixstate(module)->WaitidResultType); +#endif +#if defined(HAVE_WAIT3) || defined(HAVE_WAIT4) + Py_VISIT(_posixstate(module)->struct_rusage); +#endif + Py_VISIT(_posixstate(module)->st_mode); + return 0; +} + +static void +_posix_free(void *module) +{ + _posix_clear((PyObject *)module); +} static void fill_time(PyObject *v, int index, time_t sec, unsigned long nsec) @@ -2095,7 +2166,7 @@ fill_time(PyObject *v, int index, time_t sec, unsigned long nsec) if (!(s && ns_fractional)) goto exit; - s_in_ns = PyNumber_Multiply(s, billion); + s_in_ns = PyNumber_Multiply(s, _posixstate_global->billion); if (!s_in_ns) goto exit; @@ -2128,7 +2199,8 @@ static PyObject* _pystat_fromstructstat(STRUCT_STAT *st) { unsigned long ansec, mnsec, cnsec; - PyObject *v = PyStructSequence_New(StatResultType); + PyObject *StatResultType = _posixstate_global->StatResultType; + PyObject *v = PyStructSequence_New((PyTypeObject *)StatResultType); if (v == NULL) return NULL; @@ -4505,15 +4577,12 @@ or via the attributes sysname, nodename, release, version, and machine.\n\ See os.uname for more information."); static PyStructSequence_Desc uname_result_desc = { - "uname_result", /* name */ + MODNAME ".uname_result", /* name */ uname_result__doc__, /* doc */ uname_result_fields, 5 }; -static PyTypeObject* UnameResultType; - - #ifdef HAVE_UNAME /*[clinic input] os.uname @@ -4539,7 +4608,8 @@ os_uname_impl(PyObject *module) if (res < 0) return posix_error(); - value = PyStructSequence_New(UnameResultType); + PyObject *UnameResultType = _posixstate(module)->UnameResultType; + value = PyStructSequence_New((PyTypeObject *)UnameResultType); if (value == NULL) return NULL; @@ -4720,13 +4790,13 @@ split_py_long_to_s_and_ns(PyObject *py_long, time_t *s, long *ns) { int result = 0; PyObject *divmod; - divmod = PyNumber_Divmod(py_long, billion); + divmod = PyNumber_Divmod(py_long, _posixstate_global->billion); if (!divmod) goto exit; if (!PyTuple_Check(divmod) || PyTuple_GET_SIZE(divmod) != 2) { PyErr_Format(PyExc_TypeError, "%.200s.__divmod__() must return a 2-tuple, not %.200s", - Py_TYPE(py_long)->tp_name, Py_TYPE(divmod)->tp_name); + _PyType_Name(Py_TYPE(py_long)), _PyType_Name(Py_TYPE(divmod))); goto exit; } *s = _PyLong_AsTime_t(PyTuple_GET_ITEM(divmod, 0)); @@ -5973,7 +6043,7 @@ check_null_or_callable(PyObject *obj, const char* obj_name) { if (obj && !PyCallable_Check(obj)) { PyErr_Format(PyExc_TypeError, "'%s' must be callable, not %s", - obj_name, Py_TYPE(obj)->tp_name); + obj_name, _PyType_Name(Py_TYPE(obj))); return -1; } return 0; @@ -6177,12 +6247,12 @@ os.sched_param.__new__ sched_priority: object A scheduling parameter. -Current has only one field: sched_priority"); +Currently has only one field: sched_priority [clinic start generated code]*/ static PyObject * os_sched_param_impl(PyTypeObject *type, PyObject *sched_priority) -/*[clinic end generated code: output=48f4067d60f48c13 input=ab4de35a9a7811f2]*/ +/*[clinic end generated code: output=48f4067d60f48c13 input=eb42909a2c0e3e6c]*/ { PyObject *res; @@ -6194,7 +6264,6 @@ os_sched_param_impl(PyTypeObject *type, PyObject *sched_priority) return res; } - PyDoc_VAR(os_sched_param__doc__); static PyStructSequence_Field sched_param_fields[] = { @@ -6214,7 +6283,8 @@ convert_sched_param(PyObject *param, struct sched_param *res) { long priority; - if (Py_TYPE(param) != SchedParamType) { + PyObject *SchedParamType = _posixstate_global->SchedParamType; + if (Py_TYPE(param) != (PyTypeObject *)SchedParamType) { PyErr_SetString(PyExc_TypeError, "must have a sched_param object"); return 0; } @@ -6285,7 +6355,8 @@ os_sched_getparam_impl(PyObject *module, pid_t pid) if (sched_getparam(pid, ¶m)) return posix_error(); - result = PyStructSequence_New(SchedParamType); + PyObject *SchedParamType = _posixstate_global->SchedParamType; + result = PyStructSequence_New((PyTypeObject *)SchedParamType); if (!result) return NULL; priority = PyLong_FromLong(param.sched_priority); @@ -7494,8 +7565,7 @@ static PyObject * wait_helper(pid_t pid, int status, struct rusage *ru) { PyObject *result; - static PyObject *struct_rusage; - _Py_IDENTIFIER(struct_rusage); + PyObject *struct_rusage; if (pid == -1) return posix_error(); @@ -7506,15 +7576,13 @@ wait_helper(pid_t pid, int status, struct rusage *ru) memset(ru, 0, sizeof(*ru)); } - if (struct_rusage == NULL) { - PyObject *m = PyImport_ImportModuleNoBlock("resource"); - if (m == NULL) - return NULL; - struct_rusage = _PyObject_GetAttrId(m, &PyId_struct_rusage); - Py_DECREF(m); - if (struct_rusage == NULL) - return NULL; - } + PyObject *m = PyImport_ImportModuleNoBlock("resource"); + if (m == NULL) + return NULL; + struct_rusage = PyObject_GetAttr(m, _posixstate_global->struct_rusage); + Py_DECREF(m); + if (struct_rusage == NULL) + return NULL; /* XXX(nnorwitz): Copied (w/mods) from resource.c, there should be only one. */ result = PyStructSequence_New((PyTypeObject*) struct_rusage); @@ -7668,7 +7736,8 @@ os_waitid_impl(PyObject *module, idtype_t idtype, id_t id, int options) if (si.si_pid == 0) Py_RETURN_NONE; - result = PyStructSequence_New(WaitidResultType); + PyObject *WaitidResultType = _posixstate(module)->WaitidResultType; + result = PyStructSequence_New((PyTypeObject *)WaitidResultType); if (!result) return NULL; @@ -8119,8 +8188,6 @@ static PyStructSequence_Desc times_result_desc = { 5 }; -static PyTypeObject* TimesResultType; - #ifdef MS_WINDOWS #define HAVE_TIMES /* mandatory, for the method table */ #endif @@ -8132,7 +8199,8 @@ build_times_result(double user, double system, double children_user, double children_system, double elapsed) { - PyObject *value = PyStructSequence_New(TimesResultType); + PyObject *TimesResultType = _posixstate_global->TimesResultType; + PyObject *value = PyStructSequence_New((PyTypeObject *)TimesResultType); if (value == NULL) return NULL; @@ -9953,10 +10021,6 @@ os_posix_fadvise_impl(PyObject *module, int fd, Py_off_t offset, #ifdef HAVE_PUTENV -/* Save putenv() parameters as values here, so we can collect them when they - * get re-set with another call for the same key. */ -static PyObject *posix_putenv_garbage; - static void posix_putenv_garbage_setitem(PyObject *name, PyObject *value) { @@ -9964,7 +10028,7 @@ posix_putenv_garbage_setitem(PyObject *name, PyObject *value) * this will cause previous value to be collected. This has to * happen after the real putenv() call because the old value * was still accessible until then. */ - if (PyDict_SetItem(posix_putenv_garbage, name, value)) + if (PyDict_SetItem(_posixstate_global->posix_putenv_garbage, name, value)) /* really not much we can do; just leak */ PyErr_Clear(); else @@ -10101,7 +10165,7 @@ os_unsetenv_impl(PyObject *module, PyObject *name) * happen after the real unsetenv() call because the * old value was still accessible until then. */ - if (PyDict_DelItem(posix_putenv_garbage, name)) { + if (PyDict_DelItem(_posixstate(module)->posix_putenv_garbage, name)) { /* really not much we can do; just leak */ if (!PyErr_ExceptionMatches(PyExc_KeyError)) { return NULL; @@ -10312,7 +10376,8 @@ os_WSTOPSIG_impl(PyObject *module, int status) static PyObject* _pystatvfs_fromstructstatvfs(struct statvfs st) { - PyObject *v = PyStructSequence_New(StatVFSResultType); + PyObject *StatVFSResultType = _posixstate_global->StatVFSResultType; + PyObject *v = PyStructSequence_New((PyTypeObject *)StatVFSResultType); if (v == NULL) return NULL; @@ -12089,8 +12154,6 @@ os_memfd_create_impl(PyObject *module, PyObject *name, unsigned int flags) /* Terminal size querying */ -static PyTypeObject* TerminalSizeType; - PyDoc_STRVAR(TerminalSize_docstring, "A tuple of (columns, lines) for holding terminal window size"); @@ -12181,7 +12244,8 @@ get_terminal_size(PyObject *self, PyObject *args) } #endif /* TERMSIZE_USE_CONIO */ - termsize = PyStructSequence_New(TerminalSizeType); + PyObject *TerminalSizeType = _posixstate(self)->TerminalSizeType; + termsize = PyStructSequence_New((PyTypeObject *)TerminalSizeType); if (termsize == NULL) return NULL; PyStructSequence_SET_ITEM(termsize, 0, PyLong_FromLong(columns)); @@ -12379,9 +12443,9 @@ os_set_blocking_impl(PyObject *module, int fd, int blocking) /*[clinic input] -class os.DirEntry "DirEntry *" "&DirEntryType" +class os.DirEntry "DirEntry *" "DirEntryType" [clinic start generated code]*/ -/*[clinic end generated code: output=da39a3ee5e6b4b0d input=3138f09f7c683f1d]*/ +/*[clinic end generated code: output=da39a3ee5e6b4b0d input=3c18c7a448247980]*/ typedef struct { PyObject_HEAD @@ -12402,14 +12466,25 @@ typedef struct { #endif } DirEntry; +static PyObject * +_disabled_new(PyTypeObject *type, PyObject *args, PyObject *kwargs) +{ + PyErr_Format(PyExc_TypeError, + "cannot create '%.100s' instances", _PyType_Name(type)); + return NULL; +} + static void DirEntry_dealloc(DirEntry *entry) { + PyTypeObject *tp = Py_TYPE(entry); Py_XDECREF(entry->name); Py_XDECREF(entry->path); Py_XDECREF(entry->stat); Py_XDECREF(entry->lstat); - Py_TYPE(entry)->tp_free((PyObject *)entry); + freefunc free_func = PyType_GetSlot(tp, Py_tp_free); + free_func(entry); + Py_DECREF(tp); } /* Forward reference */ @@ -12538,7 +12613,6 @@ DirEntry_test_mode(DirEntry *self, int follow_symlinks, unsigned short mode_bits #ifdef MS_WINDOWS unsigned long dir_bits; #endif - _Py_IDENTIFIER(st_mode); #ifdef MS_WINDOWS is_symlink = (self->win32_lstat.st_mode & S_IFMT) == S_IFLNK; @@ -12561,7 +12635,7 @@ DirEntry_test_mode(DirEntry *self, int follow_symlinks, unsigned short mode_bits } goto error; } - st_mode = _PyObject_GetAttrId(stat, &PyId_st_mode); + st_mode = PyObject_GetAttr(stat, _posixstate_global->st_mode); if (!st_mode) goto error; @@ -12709,39 +12783,24 @@ static PyMethodDef DirEntry_methods[] = { {NULL} }; -static PyTypeObject DirEntryType = { - PyVarObject_HEAD_INIT(NULL, 0) - MODNAME ".DirEntry", /* tp_name */ - sizeof(DirEntry), /* tp_basicsize */ - 0, /* tp_itemsize */ - /* methods */ - (destructor)DirEntry_dealloc, /* tp_dealloc */ - 0, /* tp_vectorcall_offset */ - 0, /* tp_getattr */ - 0, /* tp_setattr */ - 0, /* tp_as_async */ - (reprfunc)DirEntry_repr, /* tp_repr */ - 0, /* tp_as_number */ - 0, /* tp_as_sequence */ - 0, /* tp_as_mapping */ - 0, /* tp_hash */ - 0, /* tp_call */ - 0, /* tp_str */ - 0, /* tp_getattro */ - 0, /* tp_setattro */ - 0, /* tp_as_buffer */ - Py_TPFLAGS_DEFAULT, /* tp_flags */ - 0, /* tp_doc */ - 0, /* tp_traverse */ - 0, /* tp_clear */ - 0, /* tp_richcompare */ - 0, /* tp_weaklistoffset */ - 0, /* tp_iter */ - 0, /* tp_iternext */ - DirEntry_methods, /* tp_methods */ - DirEntry_members, /* tp_members */ +static PyType_Slot DirEntryType_slots[] = { + {Py_tp_new, _disabled_new}, + {Py_tp_dealloc, DirEntry_dealloc}, + {Py_tp_repr, DirEntry_repr}, + {Py_tp_methods, DirEntry_methods}, + {Py_tp_members, DirEntry_members}, + {0, 0}, +}; + +static PyType_Spec DirEntryType_spec = { + MODNAME ".DirEntry", + sizeof(DirEntry), + 0, + Py_TPFLAGS_DEFAULT, + DirEntryType_slots }; + #ifdef MS_WINDOWS static wchar_t * @@ -12785,7 +12844,8 @@ DirEntry_from_find_data(path_t *path, WIN32_FIND_DATAW *dataW) ULONG reparse_tag; wchar_t *joined_path; - entry = PyObject_New(DirEntry, &DirEntryType); + PyObject *DirEntryType = _posixstate_global->DirEntryType; + entry = PyObject_New(DirEntry, (PyTypeObject *)DirEntryType); if (!entry) return NULL; entry->name = NULL; @@ -12872,7 +12932,8 @@ DirEntry_from_posix_info(path_t *path, const char *name, Py_ssize_t name_len, DirEntry *entry; char *joined_path; - entry = PyObject_New(DirEntry, &DirEntryType); + PyObject *DirEntryType = _posixstate_global->DirEntryType; + entry = PyObject_New(DirEntry, (PyTypeObject *)DirEntryType); if (!entry) return NULL; entry->name = NULL; @@ -13134,10 +13195,13 @@ ScandirIterator_finalize(ScandirIterator *iterator) static void ScandirIterator_dealloc(ScandirIterator *iterator) { + PyTypeObject *tp = Py_TYPE(iterator); if (PyObject_CallFinalizerFromDealloc((PyObject *)iterator) < 0) return; - Py_TYPE(iterator)->tp_free((PyObject *)iterator); + freefunc free_func = PyType_GetSlot(tp, Py_tp_free); + free_func(iterator); + Py_DECREF(tp); } static PyMethodDef ScandirIterator_methods[] = { @@ -13147,56 +13211,22 @@ static PyMethodDef ScandirIterator_methods[] = { {NULL} }; -static PyTypeObject ScandirIteratorType = { - PyVarObject_HEAD_INIT(NULL, 0) - MODNAME ".ScandirIterator", /* tp_name */ - sizeof(ScandirIterator), /* tp_basicsize */ - 0, /* tp_itemsize */ - /* methods */ - (destructor)ScandirIterator_dealloc, /* tp_dealloc */ - 0, /* tp_vectorcall_offset */ - 0, /* tp_getattr */ - 0, /* tp_setattr */ - 0, /* tp_as_async */ - 0, /* tp_repr */ - 0, /* tp_as_number */ - 0, /* tp_as_sequence */ - 0, /* tp_as_mapping */ - 0, /* tp_hash */ - 0, /* tp_call */ - 0, /* tp_str */ - 0, /* tp_getattro */ - 0, /* tp_setattro */ - 0, /* tp_as_buffer */ - Py_TPFLAGS_DEFAULT, /* tp_flags */ - 0, /* tp_doc */ - 0, /* tp_traverse */ - 0, /* tp_clear */ - 0, /* tp_richcompare */ - 0, /* tp_weaklistoffset */ - PyObject_SelfIter, /* tp_iter */ - (iternextfunc)ScandirIterator_iternext, /* tp_iternext */ - ScandirIterator_methods, /* tp_methods */ - 0, /* tp_members */ - 0, /* tp_getset */ - 0, /* tp_base */ - 0, /* tp_dict */ - 0, /* tp_descr_get */ - 0, /* tp_descr_set */ - 0, /* tp_dictoffset */ - 0, /* tp_init */ - 0, /* tp_alloc */ - 0, /* tp_new */ - 0, /* tp_free */ - 0, /* tp_is_gc */ - 0, /* tp_bases */ - 0, /* tp_mro */ - 0, /* tp_cache */ - 0, /* tp_subclasses */ - 0, /* tp_weaklist */ - 0, /* tp_del */ - 0, /* tp_version_tag */ - (destructor)ScandirIterator_finalize, /* tp_finalize */ +static PyType_Slot ScandirIteratorType_slots[] = { + {Py_tp_new, _disabled_new}, + {Py_tp_dealloc, ScandirIterator_dealloc}, + {Py_tp_finalize, ScandirIterator_finalize}, + {Py_tp_iter, PyObject_SelfIter}, + {Py_tp_iternext, ScandirIterator_iternext}, + {Py_tp_methods, ScandirIterator_methods}, + {0, 0}, +}; + +static PyType_Spec ScandirIteratorType_spec = { + MODNAME ".ScandirIterator", + sizeof(ScandirIterator), + 0, + Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_FINALIZE, + ScandirIteratorType_slots }; /*[clinic input] @@ -13232,7 +13262,8 @@ os_scandir_impl(PyObject *module, path_t *path) return NULL; } - iterator = PyObject_New(ScandirIterator, &ScandirIteratorType); + PyObject *ScandirIteratorType = _posixstate(module)->ScandirIteratorType; + iterator = PyObject_New(ScandirIterator, (PyTypeObject *)ScandirIteratorType); if (!iterator) return NULL; @@ -13322,7 +13353,6 @@ PyOS_FSPath(PyObject *path) { /* For error message reasons, this function is manually inlined in path_converter(). */ - _Py_IDENTIFIER(__fspath__); PyObject *func = NULL; PyObject *path_repr = NULL; @@ -13336,7 +13366,7 @@ PyOS_FSPath(PyObject *path) return PyErr_Format(PyExc_TypeError, "expected str, bytes or os.PathLike object, " "not %.200s", - Py_TYPE(path)->tp_name); + _PyType_Name(Py_TYPE(path))); } path_repr = _PyObject_CallNoArg(func); @@ -13348,8 +13378,8 @@ PyOS_FSPath(PyObject *path) if (!(PyUnicode_Check(path_repr) || PyBytes_Check(path_repr))) { PyErr_Format(PyExc_TypeError, "expected %.200s.__fspath__() to return str or bytes, " - "not %.200s", Py_TYPE(path)->tp_name, - Py_TYPE(path_repr)->tp_name); + "not %.200s", _PyType_Name(Py_TYPE(path)), + _PyType_Name(Py_TYPE(path_repr))); Py_DECREF(path_repr); return NULL; } @@ -14258,12 +14288,12 @@ static struct PyModuleDef posixmodule = { PyModuleDef_HEAD_INIT, MODNAME, posix__doc__, - -1, + sizeof(_posixstate), posix_methods, NULL, - NULL, - NULL, - NULL + _posix_traverse, + _posix_clear, + _posix_free, }; @@ -14408,6 +14438,12 @@ INITFUNC(void) PyObject *list; const char * const *trace; + m = PyState_FindModule(&posixmodule); + if (m != NULL) { + Py_INCREF(m); + return m; + } + m = PyModule_Create(&posixmodule); if (m == NULL) return NULL; @@ -14429,94 +14465,106 @@ INITFUNC(void) PyModule_AddObject(m, "error", PyExc_OSError); #ifdef HAVE_PUTENV - if (posix_putenv_garbage == NULL) - posix_putenv_garbage = PyDict_New(); + /* Save putenv() parameters as values here, so we can collect them when they + * get re-set with another call for the same key. */ + _posixstate(m)->posix_putenv_garbage = PyDict_New(); #endif - if (!initialized) { #if defined(HAVE_WAITID) && !defined(__APPLE__) - waitid_result_desc.name = MODNAME ".waitid_result"; - WaitidResultType = PyStructSequence_NewType(&waitid_result_desc); - if (WaitidResultType == NULL) { - return NULL; - } + waitid_result_desc.name = MODNAME ".waitid_result"; + PyObject *WaitidResultType = (PyObject *)PyStructSequence_NewType(&waitid_result_desc); + if (WaitidResultType == NULL) { + return NULL; + } + Py_INCREF(WaitidResultType); + PyModule_AddObject(m, "waitid_result", WaitidResultType); + _posixstate(m)->WaitidResultType = WaitidResultType; #endif - stat_result_desc.name = "os.stat_result"; /* see issue #19209 */ - stat_result_desc.fields[7].name = PyStructSequence_UnnamedField; - stat_result_desc.fields[8].name = PyStructSequence_UnnamedField; - stat_result_desc.fields[9].name = PyStructSequence_UnnamedField; - StatResultType = PyStructSequence_NewType(&stat_result_desc); - if (StatResultType == NULL) { - return NULL; - } - structseq_new = StatResultType->tp_new; - StatResultType->tp_new = statresult_new; + stat_result_desc.name = "os.stat_result"; /* see issue #19209 */ + stat_result_desc.fields[7].name = PyStructSequence_UnnamedField; + stat_result_desc.fields[8].name = PyStructSequence_UnnamedField; + stat_result_desc.fields[9].name = PyStructSequence_UnnamedField; + PyObject *StatResultType = (PyObject *)PyStructSequence_NewType(&stat_result_desc); + if (StatResultType == NULL) { + return NULL; + } + Py_INCREF(StatResultType); + PyModule_AddObject(m, "stat_result", StatResultType); + _posixstate(m)->StatResultType = StatResultType; + structseq_new = ((PyTypeObject *)StatResultType)->tp_new; + ((PyTypeObject *)StatResultType)->tp_new = statresult_new; - statvfs_result_desc.name = "os.statvfs_result"; /* see issue #19209 */ - StatVFSResultType = PyStructSequence_NewType(&statvfs_result_desc); - if (StatVFSResultType == NULL) { - return NULL; - } + statvfs_result_desc.name = "os.statvfs_result"; /* see issue #19209 */ + PyObject *StatVFSResultType = (PyObject *)PyStructSequence_NewType(&statvfs_result_desc); + if (StatVFSResultType == NULL) { + return NULL; + } + Py_INCREF(StatVFSResultType); + PyModule_AddObject(m, "statvfs_result", StatVFSResultType); + _posixstate(m)->StatVFSResultType = StatVFSResultType; #ifdef NEED_TICKS_PER_SECOND # if defined(HAVE_SYSCONF) && defined(_SC_CLK_TCK) - ticks_per_second = sysconf(_SC_CLK_TCK); + ticks_per_second = sysconf(_SC_CLK_TCK); # elif defined(HZ) - ticks_per_second = HZ; + ticks_per_second = HZ; # else - ticks_per_second = 60; /* magic fallback value; may be bogus */ + ticks_per_second = 60; /* magic fallback value; may be bogus */ # endif #endif #if defined(HAVE_SCHED_SETPARAM) || defined(HAVE_SCHED_SETSCHEDULER) || defined(POSIX_SPAWN_SETSCHEDULER) || defined(POSIX_SPAWN_SETSCHEDPARAM) - sched_param_desc.name = MODNAME ".sched_param"; - SchedParamType = PyStructSequence_NewType(&sched_param_desc); - if (SchedParamType == NULL) { - return NULL; - } - SchedParamType->tp_new = os_sched_param; + sched_param_desc.name = MODNAME ".sched_param"; + PyObject *SchedParamType = (PyObject *)PyStructSequence_NewType(&sched_param_desc); + if (SchedParamType == NULL) { + return NULL; + } + Py_INCREF(SchedParamType); + PyModule_AddObject(m, "sched_param", SchedParamType); + _posixstate(m)->SchedParamType = SchedParamType; + ((PyTypeObject *)SchedParamType)->tp_new = os_sched_param; #endif - /* initialize TerminalSize_info */ - TerminalSizeType = PyStructSequence_NewType(&TerminalSize_desc); - if (TerminalSizeType == NULL) { - return NULL; - } + /* initialize TerminalSize_info */ + PyObject *TerminalSizeType = (PyObject *)PyStructSequence_NewType(&TerminalSize_desc); + if (TerminalSizeType == NULL) { + return NULL; + } + Py_INCREF(TerminalSizeType); + PyModule_AddObject(m, "terminal_size", TerminalSizeType); + _posixstate(m)->TerminalSizeType = TerminalSizeType; - /* initialize scandir types */ - if (PyType_Ready(&ScandirIteratorType) < 0) - return NULL; - if (PyType_Ready(&DirEntryType) < 0) - return NULL; + /* initialize scandir types */ + PyObject *ScandirIteratorType = PyType_FromSpec(&ScandirIteratorType_spec); + if (ScandirIteratorType == NULL) { + return NULL; } -#if defined(HAVE_WAITID) && !defined(__APPLE__) - Py_INCREF((PyObject*) WaitidResultType); - PyModule_AddObject(m, "waitid_result", (PyObject*) WaitidResultType); -#endif - Py_INCREF((PyObject*) StatResultType); - PyModule_AddObject(m, "stat_result", (PyObject*) StatResultType); - Py_INCREF((PyObject*) StatVFSResultType); - PyModule_AddObject(m, "statvfs_result", - (PyObject*) StatVFSResultType); + _posixstate(m)->ScandirIteratorType = ScandirIteratorType; -#if defined(HAVE_SCHED_SETPARAM) || defined(HAVE_SCHED_SETSCHEDULER) - Py_INCREF(SchedParamType); - PyModule_AddObject(m, "sched_param", (PyObject *)SchedParamType); -#endif + PyObject *DirEntryType = PyType_FromSpec(&DirEntryType_spec); + if (DirEntryType == NULL) { + return NULL; + } + Py_INCREF(DirEntryType); + PyModule_AddObject(m, "DirEntry", DirEntryType); + _posixstate(m)->DirEntryType = DirEntryType; times_result_desc.name = MODNAME ".times_result"; - TimesResultType = PyStructSequence_NewType(×_result_desc); + PyObject *TimesResultType = (PyObject *)PyStructSequence_NewType(×_result_desc); if (TimesResultType == NULL) { return NULL; } - PyModule_AddObject(m, "times_result", (PyObject *)TimesResultType); + Py_INCREF(TimesResultType); + PyModule_AddObject(m, "times_result", TimesResultType); + _posixstate(m)->TimesResultType = TimesResultType; - uname_result_desc.name = MODNAME ".uname_result"; - UnameResultType = PyStructSequence_NewType(&uname_result_desc); + PyTypeObject *UnameResultType = PyStructSequence_NewType(&uname_result_desc); if (UnameResultType == NULL) { return NULL; } + Py_INCREF(UnameResultType); PyModule_AddObject(m, "uname_result", (PyObject *)UnameResultType); + _posixstate(m)->UnameResultType = (PyObject *)UnameResultType; #ifdef __APPLE__ /* @@ -14556,11 +14604,15 @@ INITFUNC(void) #endif /* __APPLE__ */ - Py_INCREF(TerminalSizeType); - PyModule_AddObject(m, "terminal_size", (PyObject*)TerminalSizeType); - - billion = PyLong_FromLong(1000000000); - if (!billion) + if ((_posixstate(m)->billion = PyLong_FromLong(1000000000)) == NULL) + return NULL; +#if defined(HAVE_WAIT3) || defined(HAVE_WAIT4) + _posixstate(m)->struct_rusage = PyUnicode_InternFromString("struct_rusage"); + if (_posixstate(m)->struct_rusage == NULL) + return NULL; +#endif + _posixstate(m)->st_mode = PyUnicode_InternFromString("st_mode"); + if (_posixstate(m)->st_mode == NULL) return NULL; /* suppress "function not used" warnings */ @@ -14590,11 +14642,6 @@ INITFUNC(void) } PyModule_AddObject(m, "_have_functions", list); - Py_INCREF((PyObject *) &DirEntryType); - PyModule_AddObject(m, "DirEntry", (PyObject *)&DirEntryType); - - initialized = 1; - return m; } From bf17d41826a8bb4bc1e34ba6345da98aac779e41 Mon Sep 17 00:00:00 2001 From: Jeroen Demeyer Date: Tue, 5 Nov 2019 16:48:04 +0100 Subject: [PATCH 13/34] bpo-37645: add new function _PyObject_FunctionStr() (GH-14890) Additional note: the `method_check_args` function in `Objects/descrobject.c` is written in such a way that it applies to all kinds of descriptors. In particular, a future re-implementation of `wrapper_descriptor` could use that code. CC @vstinner @encukou https://bugs.python.org/issue37645 Automerge-Triggered-By: @encukou --- Doc/c-api/object.rst | 1 + Include/cpython/object.h | 1 + Lib/test/test_call.py | 10 ++-- Lib/test/test_descr.py | 2 +- Lib/test/test_extcall.py | 38 ++++++------ Lib/test/test_unpack_ex.py | 10 ++-- .../2019-07-21-21-08-47.bpo-37645.4DcUaI.rst | 2 + Objects/descrobject.c | 57 +++++++++--------- Objects/methodobject.c | 37 ++++++------ Objects/object.c | 58 +++++++++++++++++++ Python/ceval.c | 49 ++++++++++------ 11 files changed, 171 insertions(+), 94 deletions(-) create mode 100644 Misc/NEWS.d/next/C API/2019-07-21-21-08-47.bpo-37645.4DcUaI.rst diff --git a/Doc/c-api/object.rst b/Doc/c-api/object.rst index 9d1155184725b3..7d7a3be02fabc2 100644 --- a/Doc/c-api/object.rst +++ b/Doc/c-api/object.rst @@ -196,6 +196,7 @@ Object Protocol This function now includes a debug assertion to help ensure that it does not silently discard an active exception. + .. c:function:: PyObject* PyObject_Bytes(PyObject *o) .. index:: builtin: bytes diff --git a/Include/cpython/object.h b/Include/cpython/object.h index fd4e77103f01b8..75e55995b57413 100644 --- a/Include/cpython/object.h +++ b/Include/cpython/object.h @@ -348,6 +348,7 @@ static inline void _Py_Dealloc_inline(PyObject *op) } #define _Py_Dealloc(op) _Py_Dealloc_inline(op) +PyAPI_FUNC(PyObject *) _PyObject_FunctionStr(PyObject *); /* Safely decref `op` and set `op` to `op2`. * diff --git a/Lib/test/test_call.py b/Lib/test/test_call.py index c233ba1351fe5a..d178aa4ec2bd2c 100644 --- a/Lib/test/test_call.py +++ b/Lib/test/test_call.py @@ -74,7 +74,7 @@ def test_varargs3_kw(self): self.assertRaisesRegex(TypeError, msg, bool, x=2) def test_varargs4_kw(self): - msg = r"^index\(\) takes no keyword arguments$" + msg = r"^list[.]index\(\) takes no keyword arguments$" self.assertRaisesRegex(TypeError, msg, [].index, x=2) def test_varargs5_kw(self): @@ -90,19 +90,19 @@ def test_varargs7_kw(self): self.assertRaisesRegex(TypeError, msg, next, x=2) def test_varargs8_kw(self): - msg = r"^pack\(\) takes no keyword arguments$" + msg = r"^_struct[.]pack\(\) takes no keyword arguments$" self.assertRaisesRegex(TypeError, msg, struct.pack, x=2) def test_varargs9_kw(self): - msg = r"^pack_into\(\) takes no keyword arguments$" + msg = r"^_struct[.]pack_into\(\) takes no keyword arguments$" self.assertRaisesRegex(TypeError, msg, struct.pack_into, x=2) def test_varargs10_kw(self): - msg = r"^index\(\) takes no keyword arguments$" + msg = r"^deque[.]index\(\) takes no keyword arguments$" self.assertRaisesRegex(TypeError, msg, collections.deque().index, x=2) def test_varargs11_kw(self): - msg = r"^pack\(\) takes no keyword arguments$" + msg = r"^Struct[.]pack\(\) takes no keyword arguments$" self.assertRaisesRegex(TypeError, msg, struct.Struct.pack, struct.Struct(""), x=2) def test_varargs12_kw(self): diff --git a/Lib/test/test_descr.py b/Lib/test/test_descr.py index 796e60a7704795..d2e121820ea5f4 100644 --- a/Lib/test/test_descr.py +++ b/Lib/test/test_descr.py @@ -1967,7 +1967,7 @@ def test_methods_in_c(self): # different error messages. set_add = set.add - expected_errmsg = "descriptor 'add' of 'set' object needs an argument" + expected_errmsg = "unbound method set.add() needs an argument" with self.assertRaises(TypeError) as cm: set_add() diff --git a/Lib/test/test_extcall.py b/Lib/test/test_extcall.py index d9dcb709f75415..4edb6680e0f989 100644 --- a/Lib/test/test_extcall.py +++ b/Lib/test/test_extcall.py @@ -52,15 +52,15 @@ >>> f(1, 2, **{'a': -1, 'b': 5}, **{'a': 4, 'c': 6}) Traceback (most recent call last): ... - TypeError: f() got multiple values for keyword argument 'a' + TypeError: test.test_extcall.f() got multiple values for keyword argument 'a' >>> f(1, 2, **{'a': -1, 'b': 5}, a=4, c=6) Traceback (most recent call last): ... - TypeError: f() got multiple values for keyword argument 'a' + TypeError: test.test_extcall.f() got multiple values for keyword argument 'a' >>> f(1, 2, a=3, **{'a': 4}, **{'a': 5}) Traceback (most recent call last): ... - TypeError: f() got multiple values for keyword argument 'a' + TypeError: test.test_extcall.f() got multiple values for keyword argument 'a' >>> f(1, 2, 3, *[4, 5], **{'a':6, 'b':7}) (1, 2, 3, 4, 5) {'a': 6, 'b': 7} >>> f(1, 2, 3, x=4, y=5, *(6, 7), **{'a':8, 'b': 9}) @@ -118,7 +118,7 @@ >>> g(*Nothing()) Traceback (most recent call last): ... - TypeError: g() argument after * must be an iterable, not Nothing + TypeError: test.test_extcall.g() argument after * must be an iterable, not Nothing >>> class Nothing: ... def __len__(self): return 5 @@ -127,7 +127,7 @@ >>> g(*Nothing()) Traceback (most recent call last): ... - TypeError: g() argument after * must be an iterable, not Nothing + TypeError: test.test_extcall.g() argument after * must be an iterable, not Nothing >>> class Nothing(): ... def __len__(self): return 5 @@ -247,17 +247,17 @@ >>> h(*h) Traceback (most recent call last): ... - TypeError: h() argument after * must be an iterable, not function + TypeError: test.test_extcall.h() argument after * must be an iterable, not function >>> h(1, *h) Traceback (most recent call last): ... - TypeError: h() argument after * must be an iterable, not function + TypeError: test.test_extcall.h() argument after * must be an iterable, not function >>> h(*[1], *h) Traceback (most recent call last): ... - TypeError: h() argument after * must be an iterable, not function + TypeError: test.test_extcall.h() argument after * must be an iterable, not function >>> dir(*h) Traceback (most recent call last): @@ -268,38 +268,38 @@ >>> nothing(*h) Traceback (most recent call last): ... - TypeError: NoneType object argument after * must be an iterable, \ + TypeError: None argument after * must be an iterable, \ not function >>> h(**h) Traceback (most recent call last): ... - TypeError: h() argument after ** must be a mapping, not function + TypeError: test.test_extcall.h() argument after ** must be a mapping, not function >>> h(**[]) Traceback (most recent call last): ... - TypeError: h() argument after ** must be a mapping, not list + TypeError: test.test_extcall.h() argument after ** must be a mapping, not list >>> h(a=1, **h) Traceback (most recent call last): ... - TypeError: h() argument after ** must be a mapping, not function + TypeError: test.test_extcall.h() argument after ** must be a mapping, not function >>> h(a=1, **[]) Traceback (most recent call last): ... - TypeError: h() argument after ** must be a mapping, not list + TypeError: test.test_extcall.h() argument after ** must be a mapping, not list >>> h(**{'a': 1}, **h) Traceback (most recent call last): ... - TypeError: h() argument after ** must be a mapping, not function + TypeError: test.test_extcall.h() argument after ** must be a mapping, not function >>> h(**{'a': 1}, **[]) Traceback (most recent call last): ... - TypeError: h() argument after ** must be a mapping, not list + TypeError: test.test_extcall.h() argument after ** must be a mapping, not list >>> dir(**h) Traceback (most recent call last): @@ -309,7 +309,7 @@ >>> nothing(**h) Traceback (most recent call last): ... - TypeError: NoneType object argument after ** must be a mapping, \ + TypeError: None argument after ** must be a mapping, \ not function >>> dir(b=1, **{'b': 1}) @@ -351,17 +351,17 @@ >>> g(**MultiDict([('x', 1), ('x', 2)])) Traceback (most recent call last): ... - TypeError: g() got multiple values for keyword argument 'x' + TypeError: test.test_extcall.g() got multiple values for keyword argument 'x' >>> g(a=3, **MultiDict([('x', 1), ('x', 2)])) Traceback (most recent call last): ... - TypeError: g() got multiple values for keyword argument 'x' + TypeError: test.test_extcall.g() got multiple values for keyword argument 'x' >>> g(**MultiDict([('a', 3)]), **MultiDict([('x', 1), ('x', 2)])) Traceback (most recent call last): ... - TypeError: g() got multiple values for keyword argument 'x' + TypeError: test.test_extcall.g() got multiple values for keyword argument 'x' Another helper function diff --git a/Lib/test/test_unpack_ex.py b/Lib/test/test_unpack_ex.py index 87fea593c0201b..46f70c2b98c709 100644 --- a/Lib/test/test_unpack_ex.py +++ b/Lib/test/test_unpack_ex.py @@ -236,27 +236,27 @@ >>> f(x=5, **{'x': 3}, y=2) Traceback (most recent call last): ... - TypeError: f() got multiple values for keyword argument 'x' + TypeError: test.test_unpack_ex.f() got multiple values for keyword argument 'x' >>> f(**{'x': 3}, x=5, y=2) Traceback (most recent call last): ... - TypeError: f() got multiple values for keyword argument 'x' + TypeError: test.test_unpack_ex.f() got multiple values for keyword argument 'x' >>> f(**{'x': 3}, **{'x': 5}, y=2) Traceback (most recent call last): ... - TypeError: f() got multiple values for keyword argument 'x' + TypeError: test.test_unpack_ex.f() got multiple values for keyword argument 'x' >>> f(x=5, **{'x': 3}, **{'x': 2}) Traceback (most recent call last): ... - TypeError: f() got multiple values for keyword argument 'x' + TypeError: test.test_unpack_ex.f() got multiple values for keyword argument 'x' >>> f(**{1: 3}, **{1: 5}) Traceback (most recent call last): ... - TypeError: f() got multiple values for keyword argument '1' + TypeError: test.test_unpack_ex.f() got multiple values for keyword argument '1' Unpacking non-sequence diff --git a/Misc/NEWS.d/next/C API/2019-07-21-21-08-47.bpo-37645.4DcUaI.rst b/Misc/NEWS.d/next/C API/2019-07-21-21-08-47.bpo-37645.4DcUaI.rst new file mode 100644 index 00000000000000..2a6efaaaeafae5 --- /dev/null +++ b/Misc/NEWS.d/next/C API/2019-07-21-21-08-47.bpo-37645.4DcUaI.rst @@ -0,0 +1,2 @@ +Add :c:func:`_PyObject_FunctionStr` to get a user-friendly string representation +of a function-like object. Patch by Jeroen Demeyer. diff --git a/Objects/descrobject.c b/Objects/descrobject.c index dbab4cd4da2c63..342b993e090390 100644 --- a/Objects/descrobject.c +++ b/Objects/descrobject.c @@ -231,45 +231,38 @@ getset_set(PyGetSetDescrObject *descr, PyObject *obj, PyObject *value) * * First, common helpers */ -static const char * -get_name(PyObject *func) { - assert(PyObject_TypeCheck(func, &PyMethodDescr_Type)); - return ((PyMethodDescrObject *)func)->d_method->ml_name; -} - -typedef void (*funcptr)(void); - static inline int method_check_args(PyObject *func, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) { assert(!PyErr_Occurred()); - assert(PyObject_TypeCheck(func, &PyMethodDescr_Type)); if (nargs < 1) { - PyErr_Format(PyExc_TypeError, - "descriptor '%.200s' of '%.100s' " - "object needs an argument", - get_name(func), PyDescr_TYPE(func)->tp_name); + PyObject *funcstr = _PyObject_FunctionStr(func); + if (funcstr != NULL) { + PyErr_Format(PyExc_TypeError, + "unbound method %U needs an argument", funcstr); + Py_DECREF(funcstr); + } return -1; } PyObject *self = args[0]; - if (!_PyObject_RealIsSubclass((PyObject *)Py_TYPE(self), - (PyObject *)PyDescr_TYPE(func))) - { - PyErr_Format(PyExc_TypeError, - "descriptor '%.200s' for '%.100s' objects " - "doesn't apply to a '%.100s' object", - get_name(func), PyDescr_TYPE(func)->tp_name, - Py_TYPE(self)->tp_name); + PyObject *dummy; + if (descr_check((PyDescrObject *)func, self, &dummy)) { return -1; } if (kwnames && PyTuple_GET_SIZE(kwnames)) { - PyErr_Format(PyExc_TypeError, - "%.200s() takes no keyword arguments", get_name(func)); + PyObject *funcstr = _PyObject_FunctionStr(func); + if (funcstr != NULL) { + PyErr_Format(PyExc_TypeError, + "%U takes no keyword arguments", funcstr); + Py_DECREF(funcstr); + } return -1; } return 0; } +typedef void (*funcptr)(void); + static inline funcptr method_enter_call(PyThreadState *tstate, PyObject *func) { @@ -387,8 +380,12 @@ method_vectorcall_NOARGS( return NULL; } if (nargs != 1) { - PyErr_Format(PyExc_TypeError, - "%.200s() takes no arguments (%zd given)", get_name(func), nargs-1); + PyObject *funcstr = _PyObject_FunctionStr(func); + if (funcstr != NULL) { + PyErr_Format(PyExc_TypeError, + "%U takes no arguments (%zd given)", funcstr, nargs-1); + Py_DECREF(funcstr); + } return NULL; } PyCFunction meth = (PyCFunction)method_enter_call(tstate, func); @@ -410,9 +407,13 @@ method_vectorcall_O( return NULL; } if (nargs != 2) { - PyErr_Format(PyExc_TypeError, - "%.200s() takes exactly one argument (%zd given)", - get_name(func), nargs-1); + PyObject *funcstr = _PyObject_FunctionStr(func); + if (funcstr != NULL) { + PyErr_Format(PyExc_TypeError, + "%U takes exactly one argument (%zd given)", + funcstr, nargs-1); + Py_DECREF(funcstr); + } return NULL; } PyCFunction meth = (PyCFunction)method_enter_call(tstate, func); diff --git a/Objects/methodobject.c b/Objects/methodobject.c index c780904d7f9aa4..8f752c610ce4b9 100644 --- a/Objects/methodobject.c +++ b/Objects/methodobject.c @@ -334,15 +334,6 @@ _PyCFunction_Fini(void) * * First, common helpers */ -static const char * -get_name(PyObject *func) -{ - assert(PyCFunction_Check(func)); - PyMethodDef *method = ((PyCFunctionObject *)func)->m_ml; - return method->ml_name; -} - -typedef void (*funcptr)(void); static inline int cfunction_check_kwargs(PyThreadState *tstate, PyObject *func, PyObject *kwnames) @@ -350,13 +341,19 @@ cfunction_check_kwargs(PyThreadState *tstate, PyObject *func, PyObject *kwnames) assert(!_PyErr_Occurred(tstate)); assert(PyCFunction_Check(func)); if (kwnames && PyTuple_GET_SIZE(kwnames)) { - _PyErr_Format(tstate, PyExc_TypeError, - "%.200s() takes no keyword arguments", get_name(func)); + PyObject *funcstr = _PyObject_FunctionStr(func); + if (funcstr != NULL) { + _PyErr_Format(tstate, PyExc_TypeError, + "%U takes no keyword arguments", funcstr); + Py_DECREF(funcstr); + } return -1; } return 0; } +typedef void (*funcptr)(void); + static inline funcptr cfunction_enter_call(PyThreadState *tstate, PyObject *func) { @@ -412,9 +409,12 @@ cfunction_vectorcall_NOARGS( } Py_ssize_t nargs = PyVectorcall_NARGS(nargsf); if (nargs != 0) { - _PyErr_Format(tstate, PyExc_TypeError, - "%.200s() takes no arguments (%zd given)", - get_name(func), nargs); + PyObject *funcstr = _PyObject_FunctionStr(func); + if (funcstr != NULL) { + _PyErr_Format(tstate, PyExc_TypeError, + "%U takes no arguments (%zd given)", funcstr, nargs); + Py_DECREF(funcstr); + } return NULL; } PyCFunction meth = (PyCFunction)cfunction_enter_call(tstate, func); @@ -436,9 +436,12 @@ cfunction_vectorcall_O( } Py_ssize_t nargs = PyVectorcall_NARGS(nargsf); if (nargs != 1) { - _PyErr_Format(tstate, PyExc_TypeError, - "%.200s() takes exactly one argument (%zd given)", - get_name(func), nargs); + PyObject *funcstr = _PyObject_FunctionStr(func); + if (funcstr != NULL) { + _PyErr_Format(tstate, PyExc_TypeError, + "%U takes exactly one argument (%zd given)", funcstr, nargs); + Py_DECREF(funcstr); + } return NULL; } PyCFunction meth = (PyCFunction)cfunction_enter_call(tstate, func); diff --git a/Objects/object.c b/Objects/object.c index 9536d467f5f2a9..3e612825c27775 100644 --- a/Objects/object.c +++ b/Objects/object.c @@ -681,6 +681,64 @@ PyObject_Bytes(PyObject *v) return PyBytes_FromObject(v); } + +/* +def _PyObject_FunctionStr(x): + try: + qualname = x.__qualname__ + except AttributeError: + return str(x) + try: + mod = x.__module__ + if mod is not None and mod != 'builtins': + return f"{x.__module__}.{qualname}()" + except AttributeError: + pass + return qualname +*/ +PyObject * +_PyObject_FunctionStr(PyObject *x) +{ + _Py_IDENTIFIER(__module__); + _Py_IDENTIFIER(__qualname__); + _Py_IDENTIFIER(builtins); + assert(!PyErr_Occurred()); + PyObject *qualname; + int ret = _PyObject_LookupAttrId(x, &PyId___qualname__, &qualname); + if (qualname == NULL) { + if (ret < 0) { + return NULL; + } + return PyObject_Str(x); + } + PyObject *module; + PyObject *result = NULL; + ret = _PyObject_LookupAttrId(x, &PyId___module__, &module); + if (module != NULL && module != Py_None) { + PyObject *builtinsname = _PyUnicode_FromId(&PyId_builtins); + if (builtinsname == NULL) { + goto done; + } + ret = PyObject_RichCompareBool(module, builtinsname, Py_NE); + if (ret < 0) { + // error + goto done; + } + if (ret > 0) { + result = PyUnicode_FromFormat("%S.%S()", module, qualname); + goto done; + } + } + else if (ret < 0) { + goto done; + } + result = PyUnicode_FromFormat("%S()", qualname); +done: + Py_DECREF(qualname); + Py_XDECREF(module); + return result; +} + /* For Python 3.0.1 and later, the old three-way comparison has been completely removed in favour of rich comparisons. PyObject_Compare() and PyObject_Cmp() are gone, and the builtin cmp function no longer exists. diff --git a/Python/ceval.c b/Python/ceval.c index 9019c785080a35..4d8f1b913c5305 100644 --- a/Python/ceval.c +++ b/Python/ceval.c @@ -5351,12 +5351,17 @@ static int check_args_iterable(PyThreadState *tstate, PyObject *func, PyObject *args) { if (args->ob_type->tp_iter == NULL && !PySequence_Check(args)) { - _PyErr_Format(tstate, PyExc_TypeError, - "%.200s%.200s argument after * " - "must be an iterable, not %.200s", - PyEval_GetFuncName(func), - PyEval_GetFuncDesc(func), - args->ob_type->tp_name); + /* check_args_iterable() may be called with a live exception: + * clear it to prevent calling _PyObject_FunctionStr() with an + * exception set. */ + PyErr_Clear(); + PyObject *funcstr = _PyObject_FunctionStr(func); + if (funcstr != NULL) { + _PyErr_Format(tstate, PyExc_TypeError, + "%U argument after * must be an iterable, not %.200s", + funcstr, Py_TYPE(args)->tp_name); + Py_DECREF(funcstr); + } return -1; } return 0; @@ -5372,24 +5377,30 @@ format_kwargs_error(PyThreadState *tstate, PyObject *func, PyObject *kwargs) * is not a mapping. */ if (_PyErr_ExceptionMatches(tstate, PyExc_AttributeError)) { - _PyErr_Format(tstate, PyExc_TypeError, - "%.200s%.200s argument after ** " - "must be a mapping, not %.200s", - PyEval_GetFuncName(func), - PyEval_GetFuncDesc(func), - kwargs->ob_type->tp_name); + PyErr_Clear(); + PyObject *funcstr = _PyObject_FunctionStr(func); + if (funcstr != NULL) { + _PyErr_Format( + tstate, PyExc_TypeError, + "%U argument after ** must be a mapping, not %.200s", + funcstr, Py_TYPE(kwargs)->tp_name); + Py_DECREF(funcstr); + } } else if (_PyErr_ExceptionMatches(tstate, PyExc_KeyError)) { PyObject *exc, *val, *tb; _PyErr_Fetch(tstate, &exc, &val, &tb); if (val && PyTuple_Check(val) && PyTuple_GET_SIZE(val) == 1) { - PyObject *key = PyTuple_GET_ITEM(val, 0); - _PyErr_Format(tstate, PyExc_TypeError, - "%.200s%.200s got multiple " - "values for keyword argument '%S'", - PyEval_GetFuncName(func), - PyEval_GetFuncDesc(func), - key); + PyErr_Clear(); + PyObject *funcstr = _PyObject_FunctionStr(func); + if (funcstr != NULL) { + PyObject *key = PyTuple_GET_ITEM(val, 0); + _PyErr_Format( + tstate, PyExc_TypeError, + "%U got multiple values for keyword argument '%S'", + funcstr, key); + Py_DECREF(funcstr); + } Py_XDECREF(exc); Py_XDECREF(val); Py_XDECREF(tb); From 56698d57691af2272f695f8c17c835ed99545cde Mon Sep 17 00:00:00 2001 From: Ammar Askar Date: Tue, 5 Nov 2019 18:29:33 -0500 Subject: [PATCH 14/34] bpo-38696: Fix usage example of HTTPStatus (GH-17066) --- Doc/library/http.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Doc/library/http.rst b/Doc/library/http.rst index 8df14578de1f1e..0e3441cbcb718c 100644 --- a/Doc/library/http.rst +++ b/Doc/library/http.rst @@ -38,7 +38,7 @@ associated messages through the :class:`http.HTTPStatus` enum: >>> HTTPStatus.OK == 200 True - >>> http.HTTPStatus.OK.value + >>> HTTPStatus.OK.value 200 >>> HTTPStatus.OK.phrase 'OK' From 6c4c45efaeb40f4f837570f57d90a0b3429c6ae9 Mon Sep 17 00:00:00 2001 From: Benjamin Peterson Date: Tue, 5 Nov 2019 19:21:29 -0800 Subject: [PATCH 15/34] bpo-38692: Add os.pidfd_open. (GH-17063) --- Doc/library/os.rst | 13 ++++++ Doc/whatsnew/3.9.rst | 3 ++ Lib/test/test_posix.py | 9 ++++ .../2019-11-05-07-18-24.bpo-38692.UpatA7.rst | 1 + Modules/clinic/posixmodule.c.h | 44 ++++++++++++++++++- Modules/posixmodule.c | 25 +++++++++++ 6 files changed, 94 insertions(+), 1 deletion(-) create mode 100644 Misc/NEWS.d/next/Library/2019-11-05-07-18-24.bpo-38692.UpatA7.rst diff --git a/Doc/library/os.rst b/Doc/library/os.rst index 8e9d9e6f034e3a..9c907a7ee5911b 100644 --- a/Doc/library/os.rst +++ b/Doc/library/os.rst @@ -3539,6 +3539,19 @@ written in Python, such as a mail server's external command delivery program. .. availability:: Unix. +.. function:: pidfd_open(pid, flags=0) + + Return a file descriptor referring to the process *pid*. This descriptor can + be used to perform process management without races and signals. The *flags* + argument is provided for future extensions; no flag values are currently + defined. + + See the :manpage:`pidfd_open(2)` man page for more details. + + .. availability:: Linux 5.3+ + .. versionadded:: 3.9 + + .. function:: plock(op) Lock program segments into memory. The value of *op* (defined in diff --git a/Doc/whatsnew/3.9.rst b/Doc/whatsnew/3.9.rst index 3cac9c5eedbc23..7e778058c8ded2 100644 --- a/Doc/whatsnew/3.9.rst +++ b/Doc/whatsnew/3.9.rst @@ -150,6 +150,9 @@ os Added :data:`~os.CLD_KILLED` and :data:`~os.CLD_STOPPED` for :attr:`si_code`. (Contributed by Dong-hee Na in :issue:`38493`.) +Exposed the Linux-specific :func:`os.pidfd_open` for process management with +file descriptors. (:issue:`38692`) + threading --------- diff --git a/Lib/test/test_posix.py b/Lib/test/test_posix.py index 17e4ded2e2d695..98a39c3f040992 100644 --- a/Lib/test/test_posix.py +++ b/Lib/test/test_posix.py @@ -1470,6 +1470,15 @@ def test_path_with_null_byte(self): open(fn, 'wb').close() self.assertRaises(ValueError, os.stat, fn_with_NUL) + @unittest.skipUnless(hasattr(os, "pidfd_open"), "pidfd_open unavailable") + def test_pidfd_open(self): + with self.assertRaises(OSError) as cm: + os.pidfd_open(-1) + if cm.exception.errno == errno.ENOSYS: + self.skipTest("system does not support pidfd_open") + self.assertEqual(cm.exception.errno, errno.EINVAL) + os.close(os.pidfd_open(os.getpid(), 0)) + class PosixGroupsTester(unittest.TestCase): def setUp(self): diff --git a/Misc/NEWS.d/next/Library/2019-11-05-07-18-24.bpo-38692.UpatA7.rst b/Misc/NEWS.d/next/Library/2019-11-05-07-18-24.bpo-38692.UpatA7.rst new file mode 100644 index 00000000000000..fd19c6f9eb52ee --- /dev/null +++ b/Misc/NEWS.d/next/Library/2019-11-05-07-18-24.bpo-38692.UpatA7.rst @@ -0,0 +1 @@ +Expose the Linux ``pidfd_open`` syscall as :func:`os.pidfd_open`. diff --git a/Modules/clinic/posixmodule.c.h b/Modules/clinic/posixmodule.c.h index 3dada674fbab0d..aa4756a620aae5 100644 --- a/Modules/clinic/posixmodule.c.h +++ b/Modules/clinic/posixmodule.c.h @@ -3963,6 +3963,44 @@ os_wait(PyObject *module, PyObject *Py_UNUSED(ignored)) #endif /* defined(HAVE_WAIT) */ +#if (defined(__linux__) && defined(__NR_pidfd_open)) + +PyDoc_STRVAR(os_pidfd_open__doc__, +"pidfd_open($module, /, pid, flags=0)\n" +"--\n" +"\n" +"Return a file descriptor referring to the process *pid*.\n" +"\n" +"The descriptor can be used to perform process management without races and\n" +"signals."); + +#define OS_PIDFD_OPEN_METHODDEF \ + {"pidfd_open", (PyCFunction)(void(*)(void))os_pidfd_open, METH_FASTCALL|METH_KEYWORDS, os_pidfd_open__doc__}, + +static PyObject * +os_pidfd_open_impl(PyObject *module, pid_t pid, unsigned int flags); + +static PyObject * +os_pidfd_open(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + PyObject *return_value = NULL; + static const char * const _keywords[] = {"pid", "flags", NULL}; + static _PyArg_Parser _parser = {"" _Py_PARSE_PID "|O&:pidfd_open", _keywords, 0}; + pid_t pid; + unsigned int flags = 0; + + if (!_PyArg_ParseStackAndKeywords(args, nargs, kwnames, &_parser, + &pid, _PyLong_UnsignedInt_Converter, &flags)) { + goto exit; + } + return_value = os_pidfd_open_impl(module, pid, flags); + +exit: + return return_value; +} + +#endif /* (defined(__linux__) && defined(__NR_pidfd_open)) */ + #if (defined(HAVE_READLINK) || defined(MS_WINDOWS)) PyDoc_STRVAR(os_readlink__doc__, @@ -8480,6 +8518,10 @@ os__remove_dll_directory(PyObject *module, PyObject *const *args, Py_ssize_t nar #define OS_WAIT_METHODDEF #endif /* !defined(OS_WAIT_METHODDEF) */ +#ifndef OS_PIDFD_OPEN_METHODDEF + #define OS_PIDFD_OPEN_METHODDEF +#endif /* !defined(OS_PIDFD_OPEN_METHODDEF) */ + #ifndef OS_READLINK_METHODDEF #define OS_READLINK_METHODDEF #endif /* !defined(OS_READLINK_METHODDEF) */ @@ -8731,4 +8773,4 @@ os__remove_dll_directory(PyObject *module, PyObject *const *args, Py_ssize_t nar #ifndef OS__REMOVE_DLL_DIRECTORY_METHODDEF #define OS__REMOVE_DLL_DIRECTORY_METHODDEF #endif /* !defined(OS__REMOVE_DLL_DIRECTORY_METHODDEF) */ -/*[clinic end generated code: output=c6e67d475eef00c4 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=51ba5b9536420cea input=a9049054013a1b77]*/ diff --git a/Modules/posixmodule.c b/Modules/posixmodule.c index 6d837c6552f490..46cf7b2f55ac18 100644 --- a/Modules/posixmodule.c +++ b/Modules/posixmodule.c @@ -7861,6 +7861,30 @@ os_wait_impl(PyObject *module) } #endif /* HAVE_WAIT */ +#if defined(__linux__) && defined(__NR_pidfd_open) +/*[clinic input] +os.pidfd_open + pid: pid_t + flags: unsigned_int = 0 + +Return a file descriptor referring to the process *pid*. + +The descriptor can be used to perform process management without races and +signals. +[clinic start generated code]*/ + +static PyObject * +os_pidfd_open_impl(PyObject *module, pid_t pid, unsigned int flags) +/*[clinic end generated code: output=5c7252698947dc41 input=c3fd99ce947ccfef]*/ +{ + int fd = syscall(__NR_pidfd_open, pid, flags); + if (fd < 0) { + return posix_error(); + } + return PyLong_FromLong(fd); +} +#endif + #if defined(HAVE_READLINK) || defined(MS_WINDOWS) /*[clinic input] @@ -13671,6 +13695,7 @@ static PyMethodDef posix_methods[] = { OS_WAIT4_METHODDEF OS_WAITID_METHODDEF OS_WAITPID_METHODDEF + OS_PIDFD_OPEN_METHODDEF OS_GETSID_METHODDEF OS_SETSID_METHODDEF OS_SETPGID_METHODDEF From 5c0c325453a175350e3c18ebb10cc10c37f9595c Mon Sep 17 00:00:00 2001 From: Benjamin Peterson Date: Tue, 5 Nov 2019 21:58:31 -0800 Subject: [PATCH 16/34] closes bpo-38713: Expose P_PIDFD in os if it's defined. (GH-17071) https://bugs.python.org/issue38713 --- Doc/library/os.rst | 12 +++++++++++- Doc/whatsnew/3.9.rst | 5 +++-- .../2019-11-05-21-22-22.bpo-38713.bmhquU.rst | 2 ++ Modules/posixmodule.c | 6 ++++++ configure | 16 ++++++++++++++-- configure.ac | 2 +- pyconfig.h.in | 3 +++ 7 files changed, 40 insertions(+), 6 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2019-11-05-21-22-22.bpo-38713.bmhquU.rst diff --git a/Doc/library/os.rst b/Doc/library/os.rst index 9c907a7ee5911b..48bd6b95a9b39e 100644 --- a/Doc/library/os.rst +++ b/Doc/library/os.rst @@ -3921,7 +3921,8 @@ written in Python, such as a mail server's external command delivery program. .. function:: waitid(idtype, id, options) Wait for the completion of one or more child processes. - *idtype* can be :data:`P_PID`, :data:`P_PGID` or :data:`P_ALL`. + *idtype* can be :data:`P_PID`, :data:`P_PGID`, :data:`P_ALL`, or + :data:`P_PIDFD` on Linux. *id* specifies the pid to wait on. *options* is constructed from the ORing of one or more of :data:`WEXITED`, :data:`WSTOPPED` or :data:`WCONTINUED` and additionally may be ORed with @@ -3946,6 +3947,15 @@ written in Python, such as a mail server's external command delivery program. .. versionadded:: 3.3 +.. data:: P_PIDFD + + This is a Linux-specific *idtype* that indicates that *id* is a file + descriptor that refers to a process. + + .. availability:: Linux 5.4+ + + .. versionadded:: 3.9 + .. data:: WEXITED WSTOPPED WNOWAIT diff --git a/Doc/whatsnew/3.9.rst b/Doc/whatsnew/3.9.rst index 7e778058c8ded2..1cd21c6ab8f71b 100644 --- a/Doc/whatsnew/3.9.rst +++ b/Doc/whatsnew/3.9.rst @@ -150,8 +150,9 @@ os Added :data:`~os.CLD_KILLED` and :data:`~os.CLD_STOPPED` for :attr:`si_code`. (Contributed by Dong-hee Na in :issue:`38493`.) -Exposed the Linux-specific :func:`os.pidfd_open` for process management with -file descriptors. (:issue:`38692`) +Exposed the Linux-specific :func:`os.pidfd_open` (:issue:`38692`) and +:data:`os.P_PIDFD` (:issue:`38713`) for process management with file +descriptors. threading --------- diff --git a/Misc/NEWS.d/next/Library/2019-11-05-21-22-22.bpo-38713.bmhquU.rst b/Misc/NEWS.d/next/Library/2019-11-05-21-22-22.bpo-38713.bmhquU.rst new file mode 100644 index 00000000000000..a22719753d7f8e --- /dev/null +++ b/Misc/NEWS.d/next/Library/2019-11-05-21-22-22.bpo-38713.bmhquU.rst @@ -0,0 +1,2 @@ +Add :data:`os.P_PIDFD` constant, which may be passed to :func:`os.waitid` to +wait on a Linux process file descriptor. diff --git a/Modules/posixmodule.c b/Modules/posixmodule.c index 46cf7b2f55ac18..f7386300c5691a 100644 --- a/Modules/posixmodule.c +++ b/Modules/posixmodule.c @@ -85,6 +85,9 @@ corresponding Unix manual entries for more information on calls."); #ifdef HAVE_SYS_WAIT_H #include /* For WNOHANG */ #endif +#ifdef HAVE_LINUX_WAIT_H +#include // For P_PIDFD +#endif #ifdef HAVE_SIGNAL_H #include @@ -14099,6 +14102,9 @@ all_ins(PyObject *m) if (PyModule_AddIntMacro(m, P_PID)) return -1; if (PyModule_AddIntMacro(m, P_PGID)) return -1; if (PyModule_AddIntMacro(m, P_ALL)) return -1; +#ifdef P_PIDFD + if (PyModule_AddIntMacro(m, P_PIDFD)) return -1; +#endif #endif #ifdef WEXITED if (PyModule_AddIntMacro(m, WEXITED)) return -1; diff --git a/configure b/configure index 50840ac1521055..44f14c3c2cfe1b 100755 --- a/configure +++ b/configure @@ -782,6 +782,7 @@ infodir docdir oldincludedir includedir +runstatedir localstatedir sharedstatedir sysconfdir @@ -895,6 +896,7 @@ datadir='${datarootdir}' sysconfdir='${prefix}/etc' sharedstatedir='${prefix}/com' localstatedir='${prefix}/var' +runstatedir='${localstatedir}/run' includedir='${prefix}/include' oldincludedir='/usr/include' docdir='${datarootdir}/doc/${PACKAGE_TARNAME}' @@ -1147,6 +1149,15 @@ do | -silent | --silent | --silen | --sile | --sil) silent=yes ;; + -runstatedir | --runstatedir | --runstatedi | --runstated \ + | --runstate | --runstat | --runsta | --runst | --runs \ + | --run | --ru | --r) + ac_prev=runstatedir ;; + -runstatedir=* | --runstatedir=* | --runstatedi=* | --runstated=* \ + | --runstate=* | --runstat=* | --runsta=* | --runst=* | --runs=* \ + | --run=* | --ru=* | --r=*) + runstatedir=$ac_optarg ;; + -sbindir | --sbindir | --sbindi | --sbind | --sbin | --sbi | --sb) ac_prev=sbindir ;; -sbindir=* | --sbindir=* | --sbindi=* | --sbind=* | --sbin=* \ @@ -1284,7 +1295,7 @@ fi for ac_var in exec_prefix prefix bindir sbindir libexecdir datarootdir \ datadir sysconfdir sharedstatedir localstatedir includedir \ oldincludedir docdir infodir htmldir dvidir pdfdir psdir \ - libdir localedir mandir + libdir localedir mandir runstatedir do eval ac_val=\$$ac_var # Remove trailing slashes. @@ -1437,6 +1448,7 @@ Fine tuning of the installation directories: --sysconfdir=DIR read-only single-machine data [PREFIX/etc] --sharedstatedir=DIR modifiable architecture-independent data [PREFIX/com] --localstatedir=DIR modifiable single-machine data [PREFIX/var] + --runstatedir=DIR modifiable per-process data [LOCALSTATEDIR/run] --libdir=DIR object code libraries [EPREFIX/lib] --includedir=DIR C header files [PREFIX/include] --oldincludedir=DIR C header files for non-gcc [/usr/include] @@ -7917,7 +7929,7 @@ sys/stat.h sys/syscall.h sys/sys_domain.h sys/termio.h sys/time.h \ sys/times.h sys/types.h sys/uio.h sys/un.h sys/utsname.h sys/wait.h pty.h \ libutil.h sys/resource.h netpacket/packet.h sysexits.h bluetooth.h \ linux/tipc.h linux/random.h spawn.h util.h alloca.h endian.h \ -sys/endian.h sys/sysmacros.h linux/memfd.h sys/memfd.h sys/mman.h +sys/endian.h sys/sysmacros.h linux/memfd.h linux/wait.h sys/memfd.h sys/mman.h do : as_ac_Header=`$as_echo "ac_cv_header_$ac_header" | $as_tr_sh` ac_fn_c_check_header_mongrel "$LINENO" "$ac_header" "$as_ac_Header" "$ac_includes_default" diff --git a/configure.ac b/configure.ac index 20d8a5239f1731..0b28dda44cdb8e 100644 --- a/configure.ac +++ b/configure.ac @@ -2161,7 +2161,7 @@ sys/stat.h sys/syscall.h sys/sys_domain.h sys/termio.h sys/time.h \ sys/times.h sys/types.h sys/uio.h sys/un.h sys/utsname.h sys/wait.h pty.h \ libutil.h sys/resource.h netpacket/packet.h sysexits.h bluetooth.h \ linux/tipc.h linux/random.h spawn.h util.h alloca.h endian.h \ -sys/endian.h sys/sysmacros.h linux/memfd.h sys/memfd.h sys/mman.h) +sys/endian.h sys/sysmacros.h linux/memfd.h linux/wait.h sys/memfd.h sys/mman.h) AC_HEADER_DIRENT AC_HEADER_MAJOR diff --git a/pyconfig.h.in b/pyconfig.h.in index 9595d7537de470..50af4c6fee43de 100644 --- a/pyconfig.h.in +++ b/pyconfig.h.in @@ -642,6 +642,9 @@ /* Define to 1 if you have the header file. */ #undef HAVE_LINUX_VM_SOCKETS_H +/* Define to 1 if you have the header file. */ +#undef HAVE_LINUX_WAIT_H + /* Define to 1 if you have the `lockf' function. */ #undef HAVE_LOCKF From 519cb8772a9745b1c7d8218cabcd2f96ceda4d62 Mon Sep 17 00:00:00 2001 From: l0rb Date: Wed, 6 Nov 2019 22:21:40 +0100 Subject: [PATCH 17/34] bpo-38716: stop rotating handlers from setting inherited namer and rotator to None (GH-17072) --- Lib/logging/handlers.py | 5 +++-- Lib/test/test_logging.py | 19 +++++++++++++++++++ .../2019-11-06-15-58-07.bpo-38716.R3uMLT.rst | 1 + 3 files changed, 23 insertions(+), 2 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2019-11-06-15-58-07.bpo-38716.R3uMLT.rst diff --git a/Lib/logging/handlers.py b/Lib/logging/handlers.py index 5641fee5735567..c1aec9880d72ab 100644 --- a/Lib/logging/handlers.py +++ b/Lib/logging/handlers.py @@ -48,6 +48,9 @@ class BaseRotatingHandler(logging.FileHandler): Not meant to be instantiated directly. Instead, use RotatingFileHandler or TimedRotatingFileHandler. """ + namer = None + rotator = None + def __init__(self, filename, mode, encoding=None, delay=False, errors=None): """ Use the specified filename for streamed logging @@ -58,8 +61,6 @@ def __init__(self, filename, mode, encoding=None, delay=False, errors=None): self.mode = mode self.encoding = encoding self.errors = errors - self.namer = None - self.rotator = None def emit(self, record): """ diff --git a/Lib/test/test_logging.py b/Lib/test/test_logging.py index 53b5bfc93f33ef..6de8803081e517 100644 --- a/Lib/test/test_logging.py +++ b/Lib/test/test_logging.py @@ -5030,6 +5030,25 @@ def namer(name): self.assertFalse(os.path.exists(namer(self.fn + ".3"))) rh.close() + def test_namer_rotator_inheritance(self): + class HandlerWithNamerAndRotator(logging.handlers.RotatingFileHandler): + def namer(self, name): + return name + ".test" + + def rotator(self, source, dest): + if os.path.exists(source): + os.rename(source, dest + ".rotated") + + rh = HandlerWithNamerAndRotator( + self.fn, backupCount=2, maxBytes=1) + self.assertEqual(rh.namer(self.fn), self.fn + ".test") + rh.emit(self.next_rec()) + self.assertLogFile(self.fn) + rh.emit(self.next_rec()) + self.assertLogFile(rh.namer(self.fn + ".1") + ".rotated") + self.assertFalse(os.path.exists(rh.namer(self.fn + ".1"))) + rh.close() + @support.requires_zlib def test_rotator(self): def namer(name): diff --git a/Misc/NEWS.d/next/Library/2019-11-06-15-58-07.bpo-38716.R3uMLT.rst b/Misc/NEWS.d/next/Library/2019-11-06-15-58-07.bpo-38716.R3uMLT.rst new file mode 100644 index 00000000000000..906eb6b6cd60ab --- /dev/null +++ b/Misc/NEWS.d/next/Library/2019-11-06-15-58-07.bpo-38716.R3uMLT.rst @@ -0,0 +1 @@ +logging: change RotatingHandler namer and rotator to class-level attributes. This stops __init__ from setting them to None in the case where a subclass defines them with eponymous methods. From 7f460494d2309ace004a400bae8fc59134dc325c Mon Sep 17 00:00:00 2001 From: Raymond Hettinger Date: Wed, 6 Nov 2019 21:50:44 -0800 Subject: [PATCH 18/34] bpo-38382: Document the early-out behavior for a zero (GH-17037) --- Doc/library/statistics.rst | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/Doc/library/statistics.rst b/Doc/library/statistics.rst index a702b2463c39b1..00c0b539edce95 100644 --- a/Doc/library/statistics.rst +++ b/Doc/library/statistics.rst @@ -164,7 +164,8 @@ However, for reading convenience, most of the examples show sorted sequences. The harmonic mean, sometimes called the subcontrary mean, is the reciprocal of the arithmetic :func:`mean` of the reciprocals of the data. For example, the harmonic mean of three values *a*, *b* and *c* - will be equivalent to ``3/(1/a + 1/b + 1/c)``. + will be equivalent to ``3/(1/a + 1/b + 1/c)``. If one of the values + is zero, the result will be zero. The harmonic mean is a type of average, a measure of the central location of the data. It is often appropriate when averaging @@ -190,6 +191,10 @@ However, for reading convenience, most of the examples show sorted sequences. :exc:`StatisticsError` is raised if *data* is empty, or any element is less than zero. + The current algorithm has an early-out when it encounters a zero + in the input. This means that the subsequent inputs are not tested + for validity. (This behavior may change in the future.) + .. versionadded:: 3.6 From 9def81aa52adc3cc89554156e40742cf17312825 Mon Sep 17 00:00:00 2001 From: Vinay Sajip Date: Thu, 7 Nov 2019 10:08:58 +0000 Subject: [PATCH 19/34] bpo-36876: Moved Parser/listnode.c statics to interpreter state. (GH-16328) --- Include/internal/pycore_pystate.h | 9 +++++++++ Parser/listnode.c | 24 ++++++++++++++---------- 2 files changed, 23 insertions(+), 10 deletions(-) diff --git a/Include/internal/pycore_pystate.h b/Include/internal/pycore_pystate.h index 91003dbead0545..eb44ae93fc4c44 100644 --- a/Include/internal/pycore_pystate.h +++ b/Include/internal/pycore_pystate.h @@ -125,6 +125,15 @@ struct _is { struct _warnings_runtime_state warnings; PyObject *audit_hooks; +/* + * See bpo-36876: miscellaneous ad hoc statics have been moved here. + */ + struct { + struct { + int level; + int atbol; + } listnode; + } parser; }; PyAPI_FUNC(struct _is*) _PyInterpreterState_LookUpID(PY_INT64_T); diff --git a/Parser/listnode.c b/Parser/listnode.c index 8f1a1163b63d5c..d431ae537e3b41 100644 --- a/Parser/listnode.c +++ b/Parser/listnode.c @@ -2,6 +2,7 @@ /* List a node on a file */ #include "Python.h" +#include "pycore_pystate.h" #include "token.h" #include "node.h" @@ -15,19 +16,21 @@ PyNode_ListTree(node *n) listnode(stdout, n); } -static int level, atbol; - static void listnode(FILE *fp, node *n) { - level = 0; - atbol = 1; + PyInterpreterState *interp = _PyInterpreterState_GET_UNSAFE(); + + interp->parser.listnode.level = 0; + interp->parser.listnode.atbol = 1; list1node(fp, n); } static void list1node(FILE *fp, node *n) { + PyInterpreterState *interp; + if (n == NULL) return; if (ISNONTERMINAL(TYPE(n))) { @@ -36,25 +39,26 @@ list1node(FILE *fp, node *n) list1node(fp, CHILD(n, i)); } else if (ISTERMINAL(TYPE(n))) { + interp = _PyInterpreterState_GET_UNSAFE(); switch (TYPE(n)) { case INDENT: - ++level; + interp->parser.listnode.level++; break; case DEDENT: - --level; + interp->parser.listnode.level--; break; default: - if (atbol) { + if (interp->parser.listnode.atbol) { int i; - for (i = 0; i < level; ++i) + for (i = 0; i < interp->parser.listnode.level; ++i) fprintf(fp, "\t"); - atbol = 0; + interp->parser.listnode.atbol = 0; } if (TYPE(n) == NEWLINE) { if (STR(n) != NULL) fprintf(fp, "%s", STR(n)); fprintf(fp, "\n"); - atbol = 1; + interp->parser.listnode.atbol = 1; } else fprintf(fp, "%s ", STR(n)); From 991b02dc871e101e98edece37d8a570f6a39d79f Mon Sep 17 00:00:00 2001 From: l0rb Date: Thu, 7 Nov 2019 11:13:36 +0100 Subject: [PATCH 20/34] update a deprecated assert in logging tests (GH-17079) --- Lib/test/test_logging.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Lib/test/test_logging.py b/Lib/test/test_logging.py index 6de8803081e517..c47ad4ac752062 100644 --- a/Lib/test/test_logging.py +++ b/Lib/test/test_logging.py @@ -1613,7 +1613,7 @@ def test_config_set_handler_names(self): format=%(levelname)s ++ %(message)s """ self.apply_config(test_config) - self.assertEquals(logging.getLogger().handlers[0].name, 'hand1') + self.assertEqual(logging.getLogger().handlers[0].name, 'hand1') def test_defaults_do_no_interpolation(self): """bpo-33802 defaults should not get interpolated""" From d12d0e7c0fe2b49c40ac4d66365147c619d6c475 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Thu, 7 Nov 2019 12:42:07 +0100 Subject: [PATCH 21/34] bpo-38733: PyErr_Occurred() caller must hold the GIL (GH-17080) bpo-3605, bpo-38733: Optimize _PyErr_Occurred(): remove "tstate == NULL" test. Py_FatalError() no longer calls PyErr_Occurred() if called without holding the GIL. So PyErr_Occurred() no longer has to support tstate==NULL case. _Py_CheckFunctionResult(): use directly _PyErr_Occurred() to avoid explicit "!= NULL" test. --- Doc/c-api/exceptions.rst | 2 ++ Include/internal/pycore_pyerrors.h | 3 ++- Objects/call.c | 6 ++---- Objects/obmalloc.c | 5 +++-- Python/errors.c | 3 +++ 5 files changed, 12 insertions(+), 7 deletions(-) diff --git a/Doc/c-api/exceptions.rst b/Doc/c-api/exceptions.rst index a042c6eee0a298..cd6df00aeb5cdb 100644 --- a/Doc/c-api/exceptions.rst +++ b/Doc/c-api/exceptions.rst @@ -374,6 +374,8 @@ Querying the error indicator own a reference to the return value, so you do not need to :c:func:`Py_DECREF` it. + The caller must hold the GIL. + .. note:: Do not compare the return value to a specific exception; use diff --git a/Include/internal/pycore_pyerrors.h b/Include/internal/pycore_pyerrors.h index edbfdfa597e1f0..f3aa3e8b98abe7 100644 --- a/Include/internal/pycore_pyerrors.h +++ b/Include/internal/pycore_pyerrors.h @@ -10,7 +10,8 @@ extern "C" { static inline PyObject* _PyErr_Occurred(PyThreadState *tstate) { - return tstate == NULL ? NULL : tstate->curexc_type; + assert(tstate != NULL); + return tstate->curexc_type; } diff --git a/Objects/call.c b/Objects/call.c index 0d5c41295c29c8..a1d0b332cefcce 100644 --- a/Objects/call.c +++ b/Objects/call.c @@ -30,12 +30,10 @@ PyObject* _Py_CheckFunctionResult(PyThreadState *tstate, PyObject *callable, PyObject *result, const char *where) { - int err_occurred = (_PyErr_Occurred(tstate) != NULL); - assert((callable != NULL) ^ (where != NULL)); if (result == NULL) { - if (!err_occurred) { + if (!_PyErr_Occurred(tstate)) { if (callable) _PyErr_Format(tstate, PyExc_SystemError, "%R returned NULL without setting an error", @@ -52,7 +50,7 @@ _Py_CheckFunctionResult(PyThreadState *tstate, PyObject *callable, } } else { - if (err_occurred) { + if (_PyErr_Occurred(tstate)) { Py_DECREF(result); if (callable) { diff --git a/Objects/obmalloc.c b/Objects/obmalloc.c index 50701dbd384546..722e91e3db4402 100644 --- a/Objects/obmalloc.c +++ b/Objects/obmalloc.c @@ -2313,12 +2313,13 @@ _PyMem_DebugRawRealloc(void *ctx, void *p, size_t nbytes) return data; } -static void +static inline void _PyMem_DebugCheckGIL(void) { - if (!PyGILState_Check()) + if (!PyGILState_Check()) { Py_FatalError("Python memory allocator called " "without holding the GIL"); + } } static void * diff --git a/Python/errors.c b/Python/errors.c index 9658afeb9f773f..1783084c336b19 100644 --- a/Python/errors.c +++ b/Python/errors.c @@ -218,6 +218,9 @@ PyErr_SetString(PyObject *exception, const char *string) PyObject* _Py_HOT_FUNCTION PyErr_Occurred(void) { + /* The caller must hold the GIL. */ + assert(PyGILState_Check()); + PyThreadState *tstate = _PyThreadState_GET(); return _PyErr_Occurred(tstate); } From 6cbc84fb99acb33dd659d7adb29a20adbe62b74a Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Fri, 8 Nov 2019 00:59:04 +0900 Subject: [PATCH 22/34] bpo-38613: Optimize set operations of dict keys. (GH-16961) --- .../2019-10-29-15-44-24.bpo-38613.V_R3NC.rst | 3 ++ Objects/dictobject.c | 54 +++++++++++-------- 2 files changed, 36 insertions(+), 21 deletions(-) create mode 100644 Misc/NEWS.d/next/Core and Builtins/2019-10-29-15-44-24.bpo-38613.V_R3NC.rst diff --git a/Misc/NEWS.d/next/Core and Builtins/2019-10-29-15-44-24.bpo-38613.V_R3NC.rst b/Misc/NEWS.d/next/Core and Builtins/2019-10-29-15-44-24.bpo-38613.V_R3NC.rst new file mode 100644 index 00000000000000..c001db6679831b --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2019-10-29-15-44-24.bpo-38613.V_R3NC.rst @@ -0,0 +1,3 @@ +Optimized some set operations (e.g. ``|``, ``^``, and ``-``) of +``dict_keys``. ``d.keys() | other`` was slower than ``set(d) | other`` but +they are almost same performance for now. diff --git a/Objects/dictobject.c b/Objects/dictobject.c index d909f220a984b1..4afa19c8a0a90b 100644 --- a/Objects/dictobject.c +++ b/Objects/dictobject.c @@ -4162,17 +4162,34 @@ static PySequenceMethods dictkeys_as_sequence = { (objobjproc)dictkeys_contains, /* sq_contains */ }; +// Create an set object from dictviews object. +// Returns a new reference. +// This utility function is used by set operations. static PyObject* -dictviews_sub(PyObject* self, PyObject *other) +dictviews_to_set(PyObject *self) { - PyObject *result = PySet_New(self); - PyObject *tmp; - _Py_IDENTIFIER(difference_update); + PyObject *left = self; + if (PyDictKeys_Check(self)) { + // PySet_New() has fast path for the dict object. + PyObject *dict = (PyObject *)((_PyDictViewObject *)self)->dv_dict; + if (PyDict_CheckExact(dict)) { + left = dict; + } + } + return PySet_New(left); +} - if (result == NULL) +static PyObject* +dictviews_sub(PyObject *self, PyObject *other) +{ + PyObject *result = dictviews_to_set(self); + if (result == NULL) { return NULL; + } - tmp = _PyObject_CallMethodIdOneArg(result, &PyId_difference_update, other); + _Py_IDENTIFIER(difference_update); + PyObject *tmp = _PyObject_CallMethodIdOneArg( + result, &PyId_difference_update, other); if (tmp == NULL) { Py_DECREF(result); return NULL; @@ -4273,34 +4290,29 @@ _PyDictView_Intersect(PyObject* self, PyObject *other) static PyObject* dictviews_or(PyObject* self, PyObject *other) { - PyObject *result = PySet_New(self); - PyObject *tmp; - _Py_IDENTIFIER(update); - - if (result == NULL) + PyObject *result = dictviews_to_set(self); + if (result == NULL) { return NULL; + } - tmp = _PyObject_CallMethodIdOneArg(result, &PyId_update, other); - if (tmp == NULL) { + if (_PySet_Update(result, other) < 0) { Py_DECREF(result); return NULL; } - - Py_DECREF(tmp); return result; } static PyObject* dictviews_xor(PyObject* self, PyObject *other) { - PyObject *result = PySet_New(self); - PyObject *tmp; - _Py_IDENTIFIER(symmetric_difference_update); - - if (result == NULL) + PyObject *result = dictviews_to_set(self); + if (result == NULL) { return NULL; + } - tmp = _PyObject_CallMethodIdOneArg(result, &PyId_symmetric_difference_update, other); + _Py_IDENTIFIER(symmetric_difference_update); + PyObject *tmp = _PyObject_CallMethodIdOneArg( + result, &PyId_symmetric_difference_update, other); if (tmp == NULL) { Py_DECREF(result); return NULL; From befa032d8869e0fab4732d910f3887642879d644 Mon Sep 17 00:00:00 2001 From: Dong-hee Na Date: Fri, 8 Nov 2019 05:31:41 +0900 Subject: [PATCH 23/34] bpo-22367: Add tests for fcntl.lockf(). (GH-17010) --- Lib/test/test_fcntl.py | 29 ++++++++++++++++++++++++++++- 1 file changed, 28 insertions(+), 1 deletion(-) diff --git a/Lib/test/test_fcntl.py b/Lib/test/test_fcntl.py index a2b5997067539e..9d1be28c6d3992 100644 --- a/Lib/test/test_fcntl.py +++ b/Lib/test/test_fcntl.py @@ -5,6 +5,7 @@ import struct import sys import unittest +from multiprocessing import Process from test.support import (verbose, TESTFN, unlink, run_unittest, import_module, cpython_only) @@ -12,7 +13,6 @@ fcntl = import_module('fcntl') -# TODO - Write tests for flock() and lockf(). def get_lockdata(): try: @@ -138,6 +138,33 @@ def test_flock(self): self.assertRaises(ValueError, fcntl.flock, -1, fcntl.LOCK_SH) self.assertRaises(TypeError, fcntl.flock, 'spam', fcntl.LOCK_SH) + def test_lockf_exclusive(self): + self.f = open(TESTFN, 'wb+') + cmd = fcntl.LOCK_EX | fcntl.LOCK_NB + def try_lockf_on_other_process(): + self.assertRaises(BlockingIOError, fcntl.lockf, self.f, cmd) + + fcntl.lockf(self.f, cmd) + p = Process(target=try_lockf_on_other_process) + p.start() + p.join() + fcntl.lockf(self.f, fcntl.LOCK_UN) + self.assertEqual(p.exitcode, 0) + + def test_lockf_share(self): + self.f = open(TESTFN, 'wb+') + cmd = fcntl.LOCK_SH | fcntl.LOCK_NB + def try_lockf_on_other_process(): + fcntl.lockf(self.f, cmd) + fcntl.lockf(self.f, fcntl.LOCK_UN) + + fcntl.lockf(self.f, cmd) + p = Process(target=try_lockf_on_other_process) + p.start() + p.join() + fcntl.lockf(self.f, fcntl.LOCK_UN) + self.assertEqual(p.exitcode, 0) + @cpython_only def test_flock_overflow(self): import _testcapi From 7e433733175e76627d46ed9bdab543860cd1452d Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Fri, 8 Nov 2019 10:05:17 +0100 Subject: [PATCH 24/34] bpo-38644: Add _PyObject_VectorcallTstate() (GH-17052) * Add _PyObject_VectorcallTstate() function: similar to _PyObject_Vectorcall(), but with tstate parameter * Add tstate parameter to _PyObject_MakeTpCall() --- Include/cpython/abstract.h | 18 ++++++++++---- Modules/_functoolsmodule.c | 21 +++++++++++------ Objects/call.c | 12 ++++++---- Objects/classobject.c | 19 +++++++++------ Objects/typeobject.c | 26 +++++++++++++-------- Python/context.c | 48 ++++++++++++++++++++++++-------------- 6 files changed, 94 insertions(+), 50 deletions(-) diff --git a/Include/cpython/abstract.h b/Include/cpython/abstract.h index be37e1971a814e..fef538e63e0422 100644 --- a/Include/cpython/abstract.h +++ b/Include/cpython/abstract.h @@ -49,6 +49,7 @@ PyAPI_FUNC(PyObject *) _Py_CheckFunctionResult( or _PyObject_FastCallDict() (both forms are supported), except that nargs is plainly the number of arguments without flags. */ PyAPI_FUNC(PyObject *) _PyObject_MakeTpCall( + PyThreadState *tstate, PyObject *callable, PyObject *const *args, Py_ssize_t nargs, PyObject *keywords); @@ -95,22 +96,31 @@ _PyVectorcall_Function(PyObject *callable) Return the result on success. Raise an exception and return NULL on error. */ static inline PyObject * -_PyObject_Vectorcall(PyObject *callable, PyObject *const *args, - size_t nargsf, PyObject *kwnames) +_PyObject_VectorcallTstate(PyThreadState *tstate, PyObject *callable, + PyObject *const *args, size_t nargsf, + PyObject *kwnames) { assert(kwnames == NULL || PyTuple_Check(kwnames)); assert(args != NULL || PyVectorcall_NARGS(nargsf) == 0); - PyThreadState *tstate = PyThreadState_GET(); vectorcallfunc func = _PyVectorcall_Function(callable); if (func == NULL) { Py_ssize_t nargs = PyVectorcall_NARGS(nargsf); - return _PyObject_MakeTpCall(callable, args, nargs, kwnames); + return _PyObject_MakeTpCall(tstate, callable, args, nargs, kwnames); } PyObject *res = func(callable, args, nargsf, kwnames); return _Py_CheckFunctionResult(tstate, callable, res, NULL); } +static inline PyObject * +_PyObject_Vectorcall(PyObject *callable, PyObject *const *args, + size_t nargsf, PyObject *kwnames) +{ + PyThreadState *tstate = PyThreadState_GET(); + return _PyObject_VectorcallTstate(tstate, callable, + args, nargsf, kwnames); +} + /* Same as _PyObject_Vectorcall except that keyword arguments are passed as dict, which may be NULL if there are no keyword arguments. */ PyAPI_FUNC(PyObject *) _PyObject_FastCallDict( diff --git a/Modules/_functoolsmodule.c b/Modules/_functoolsmodule.c index 17d49ac5b342da..987087b1ac97ba 100644 --- a/Modules/_functoolsmodule.c +++ b/Modules/_functoolsmodule.c @@ -132,21 +132,25 @@ partial_dealloc(partialobject *pto) * if we would need to do that, we stop using vectorcall and fall back * to using partial_call() instead. */ _Py_NO_INLINE static PyObject * -partial_vectorcall_fallback(partialobject *pto, PyObject *const *args, - size_t nargsf, PyObject *kwnames) +partial_vectorcall_fallback(PyThreadState *tstate, partialobject *pto, + PyObject *const *args, size_t nargsf, + PyObject *kwnames) { pto->vectorcall = NULL; Py_ssize_t nargs = PyVectorcall_NARGS(nargsf); - return _PyObject_MakeTpCall((PyObject *)pto, args, nargs, kwnames); + return _PyObject_MakeTpCall(tstate, (PyObject *)pto, + args, nargs, kwnames); } static PyObject * partial_vectorcall(partialobject *pto, PyObject *const *args, size_t nargsf, PyObject *kwnames) { + PyThreadState *tstate = _PyThreadState_GET(); + /* pto->kw is mutable, so need to check every time */ if (PyDict_GET_SIZE(pto->kw)) { - return partial_vectorcall_fallback(pto, args, nargsf, kwnames); + return partial_vectorcall_fallback(tstate, pto, args, nargsf, kwnames); } Py_ssize_t nargs = PyVectorcall_NARGS(nargsf); @@ -160,7 +164,8 @@ partial_vectorcall(partialobject *pto, PyObject *const *args, /* Fast path if we're called without arguments */ if (nargs_total == 0) { - return _PyObject_Vectorcall(pto->fn, pto_args, pto_nargs, NULL); + return _PyObject_VectorcallTstate(tstate, pto->fn, + pto_args, pto_nargs, NULL); } /* Fast path using PY_VECTORCALL_ARGUMENTS_OFFSET to prepend a single @@ -169,7 +174,8 @@ partial_vectorcall(partialobject *pto, PyObject *const *args, PyObject **newargs = (PyObject **)args - 1; PyObject *tmp = newargs[0]; newargs[0] = pto_args[0]; - PyObject *ret = _PyObject_Vectorcall(pto->fn, newargs, nargs + 1, kwnames); + PyObject *ret = _PyObject_VectorcallTstate(tstate, pto->fn, + newargs, nargs + 1, kwnames); newargs[0] = tmp; return ret; } @@ -195,7 +201,8 @@ partial_vectorcall(partialobject *pto, PyObject *const *args, memcpy(stack, pto_args, pto_nargs * sizeof(PyObject*)); memcpy(stack + pto_nargs, args, nargs_total * sizeof(PyObject*)); - ret = _PyObject_Vectorcall(pto->fn, stack, pto_nargs + nargs, kwnames); + ret = _PyObject_VectorcallTstate(tstate, pto->fn, + stack, pto_nargs + nargs, kwnames); if (stack != small_stack) { PyMem_Free(stack); } diff --git a/Objects/call.c b/Objects/call.c index a1d0b332cefcce..a8ae41a7842aef 100644 --- a/Objects/call.c +++ b/Objects/call.c @@ -104,7 +104,7 @@ _PyObject_FastCallDict(PyObject *callable, PyObject *const *args, vectorcallfunc func = _PyVectorcall_Function(callable); if (func == NULL) { /* Use tp_call instead */ - return _PyObject_MakeTpCall(callable, args, nargs, kwargs); + return _PyObject_MakeTpCall(tstate, callable, args, nargs, kwargs); } PyObject *res; @@ -129,10 +129,10 @@ _PyObject_FastCallDict(PyObject *callable, PyObject *const *args, PyObject * -_PyObject_MakeTpCall(PyObject *callable, PyObject *const *args, Py_ssize_t nargs, PyObject *keywords) +_PyObject_MakeTpCall(PyThreadState *tstate, PyObject *callable, + PyObject *const *args, Py_ssize_t nargs, + PyObject *keywords) { - PyThreadState *tstate = _PyThreadState_GET(); - /* Slow path: build a temporary tuple for positional arguments and a * temporary dictionary for keyword arguments (if any) */ ternaryfunc call = Py_TYPE(callable)->tp_call; @@ -774,6 +774,7 @@ _PyObject_VectorcallMethod(PyObject *name, PyObject *const *args, assert(args != NULL); assert(PyVectorcall_NARGS(nargsf) >= 1); + PyThreadState *tstate = _PyThreadState_GET(); PyObject *callable = NULL; /* Use args[0] as "self" argument */ int unbound = _PyObject_GetMethod(args[0], name, &callable); @@ -792,7 +793,8 @@ _PyObject_VectorcallMethod(PyObject *name, PyObject *const *args, args++; nargsf--; } - PyObject *result = _PyObject_Vectorcall(callable, args, nargsf, kwnames); + PyObject *result = _PyObject_VectorcallTstate(tstate, callable, + args, nargsf, kwnames); Py_DECREF(callable); return result; } diff --git a/Objects/classobject.c b/Objects/classobject.c index 4a9add1229e374..d3fc726415406d 100644 --- a/Objects/classobject.c +++ b/Objects/classobject.c @@ -2,6 +2,7 @@ #include "Python.h" #include "pycore_object.h" +#include "pycore_pyerrors.h" #include "pycore_pymem.h" #include "pycore_pystate.h" #include "structmember.h" @@ -37,25 +38,28 @@ method_vectorcall(PyObject *method, PyObject *const *args, size_t nargsf, PyObject *kwnames) { assert(Py_TYPE(method) == &PyMethod_Type); - PyObject *self, *func, *result; - self = PyMethod_GET_SELF(method); - func = PyMethod_GET_FUNCTION(method); + + PyThreadState *tstate = _PyThreadState_GET(); + PyObject *self = PyMethod_GET_SELF(method); + PyObject *func = PyMethod_GET_FUNCTION(method); Py_ssize_t nargs = PyVectorcall_NARGS(nargsf); + PyObject *result; if (nargsf & PY_VECTORCALL_ARGUMENTS_OFFSET) { /* PY_VECTORCALL_ARGUMENTS_OFFSET is set, so we are allowed to mutate the vector */ PyObject **newargs = (PyObject**)args - 1; nargs += 1; PyObject *tmp = newargs[0]; newargs[0] = self; - result = _PyObject_Vectorcall(func, newargs, nargs, kwnames); + result = _PyObject_VectorcallTstate(tstate, func, newargs, + nargs, kwnames); newargs[0] = tmp; } else { Py_ssize_t nkwargs = (kwnames == NULL) ? 0 : PyTuple_GET_SIZE(kwnames); Py_ssize_t totalargs = nargs + nkwargs; if (totalargs == 0) { - return _PyObject_Vectorcall(func, &self, 1, NULL); + return _PyObject_VectorcallTstate(tstate, func, &self, 1, NULL); } PyObject *newargs_stack[_PY_FASTCALL_SMALL_STACK]; @@ -66,7 +70,7 @@ method_vectorcall(PyObject *method, PyObject *const *args, else { newargs = PyMem_Malloc((totalargs+1) * sizeof(PyObject *)); if (newargs == NULL) { - PyErr_NoMemory(); + _PyErr_NoMemory(tstate); return NULL; } } @@ -77,7 +81,8 @@ method_vectorcall(PyObject *method, PyObject *const *args, * undefined behaviour. */ assert(args != NULL); memcpy(newargs + 1, args, totalargs * sizeof(PyObject *)); - result = _PyObject_Vectorcall(func, newargs, nargs+1, kwnames); + result = _PyObject_VectorcallTstate(tstate, func, + newargs, nargs+1, kwnames); if (newargs != newargs_stack) { PyMem_Free(newargs); } diff --git a/Objects/typeobject.c b/Objects/typeobject.c index 0e1cb7b7aeb369..50a3c15785a900 100644 --- a/Objects/typeobject.c +++ b/Objects/typeobject.c @@ -1445,7 +1445,7 @@ lookup_method(PyObject *self, _Py_Identifier *attrid, int *unbound) static inline PyObject* -vectorcall_unbound(int unbound, PyObject *func, +vectorcall_unbound(PyThreadState *tstate, int unbound, PyObject *func, PyObject *const *args, Py_ssize_t nargs) { size_t nargsf = nargs; @@ -1455,7 +1455,7 @@ vectorcall_unbound(int unbound, PyObject *func, args++; nargsf = nargsf - 1 + PY_VECTORCALL_ARGUMENTS_OFFSET; } - return _PyObject_Vectorcall(func, args, nargsf, NULL); + return _PyObject_VectorcallTstate(tstate, func, args, nargsf, NULL); } static PyObject* @@ -1479,13 +1479,15 @@ vectorcall_method(_Py_Identifier *name, PyObject *const *args, Py_ssize_t nargs) { assert(nargs >= 1); + + PyThreadState *tstate = _PyThreadState_GET(); int unbound; PyObject *self = args[0]; PyObject *func = lookup_method(self, name, &unbound); if (func == NULL) { return NULL; } - PyObject *retval = vectorcall_unbound(unbound, func, args, nargs); + PyObject *retval = vectorcall_unbound(tstate, unbound, func, args, nargs); Py_DECREF(func); return retval; } @@ -1493,10 +1495,11 @@ vectorcall_method(_Py_Identifier *name, /* Clone of vectorcall_method() that returns NotImplemented * when the lookup fails. */ static PyObject * -vectorcall_maybe(_Py_Identifier *name, +vectorcall_maybe(PyThreadState *tstate, _Py_Identifier *name, PyObject *const *args, Py_ssize_t nargs) { assert(nargs >= 1); + int unbound; PyObject *self = args[0]; PyObject *func = lookup_maybe_method(self, name, &unbound); @@ -1505,7 +1508,7 @@ vectorcall_maybe(_Py_Identifier *name, Py_RETURN_NOTIMPLEMENTED; return NULL; } - PyObject *retval = vectorcall_unbound(unbound, func, args, nargs); + PyObject *retval = vectorcall_unbound(tstate, unbound, func, args, nargs); Py_DECREF(func); return retval; } @@ -6177,6 +6180,7 @@ static PyObject * \ FUNCNAME(PyObject *self, PyObject *other) \ { \ PyObject* stack[2]; \ + PyThreadState *tstate = _PyThreadState_GET(); \ _Py_static_string(op_id, OPSTR); \ _Py_static_string(rop_id, ROPSTR); \ int do_other = Py_TYPE(self) != Py_TYPE(other) && \ @@ -6193,7 +6197,7 @@ FUNCNAME(PyObject *self, PyObject *other) \ if (ok) { \ stack[0] = other; \ stack[1] = self; \ - r = vectorcall_maybe(&rop_id, stack, 2); \ + r = vectorcall_maybe(tstate, &rop_id, stack, 2); \ if (r != Py_NotImplemented) \ return r; \ Py_DECREF(r); \ @@ -6202,7 +6206,7 @@ FUNCNAME(PyObject *self, PyObject *other) \ } \ stack[0] = self; \ stack[1] = other; \ - r = vectorcall_maybe(&op_id, stack, 2); \ + r = vectorcall_maybe(tstate, &op_id, stack, 2); \ if (r != Py_NotImplemented || \ Py_TYPE(other) == Py_TYPE(self)) \ return r; \ @@ -6211,7 +6215,7 @@ FUNCNAME(PyObject *self, PyObject *other) \ if (do_other) { \ stack[0] = other; \ stack[1] = self; \ - return vectorcall_maybe(&rop_id, stack, 2); \ + return vectorcall_maybe(tstate, &rop_id, stack, 2); \ } \ Py_RETURN_NOTIMPLEMENTED; \ } @@ -6293,6 +6297,7 @@ slot_sq_ass_item(PyObject *self, Py_ssize_t index, PyObject *value) static int slot_sq_contains(PyObject *self, PyObject *value) { + PyThreadState *tstate = _PyThreadState_GET(); PyObject *func, *res; int result = -1, unbound; _Py_IDENTIFIER(__contains__); @@ -6307,7 +6312,7 @@ slot_sq_contains(PyObject *self, PyObject *value) } if (func != NULL) { PyObject *args[2] = {self, value}; - res = vectorcall_unbound(unbound, func, args, 2); + res = vectorcall_unbound(tstate, unbound, func, args, 2); Py_DECREF(func); if (res != NULL) { result = PyObject_IsTrue(res); @@ -6682,6 +6687,7 @@ static _Py_Identifier name_op[] = { static PyObject * slot_tp_richcompare(PyObject *self, PyObject *other, int op) { + PyThreadState *tstate = _PyThreadState_GET(); int unbound; PyObject *func, *res; @@ -6692,7 +6698,7 @@ slot_tp_richcompare(PyObject *self, PyObject *other, int op) } PyObject *stack[2] = {self, other}; - res = vectorcall_unbound(unbound, func, stack, 2); + res = vectorcall_unbound(tstate, unbound, func, stack, 2); Py_DECREF(func); return res; } diff --git a/Python/context.c b/Python/context.c index f48c376b4ffaa9..26f22994eecb7a 100644 --- a/Python/context.c +++ b/Python/context.c @@ -3,6 +3,7 @@ #include "pycore_context.h" #include "pycore_hamt.h" #include "pycore_object.h" +#include "pycore_pyerrors.h" #include "pycore_pystate.h" #include "structmember.h" @@ -101,21 +102,18 @@ PyContext_CopyCurrent(void) } -int -PyContext_Enter(PyObject *octx) +static int +_PyContext_Enter(PyThreadState *ts, PyObject *octx) { ENSURE_Context(octx, -1) PyContext *ctx = (PyContext *)octx; if (ctx->ctx_entered) { - PyErr_Format(PyExc_RuntimeError, - "cannot enter context: %R is already entered", ctx); + _PyErr_Format(ts, PyExc_RuntimeError, + "cannot enter context: %R is already entered", ctx); return -1; } - PyThreadState *ts = _PyThreadState_GET(); - assert(ts != NULL); - ctx->ctx_prev = (PyContext *)ts->context; /* borrow */ ctx->ctx_entered = 1; @@ -128,7 +126,16 @@ PyContext_Enter(PyObject *octx) int -PyContext_Exit(PyObject *octx) +PyContext_Enter(PyObject *octx) +{ + PyThreadState *ts = _PyThreadState_GET(); + assert(ts != NULL); + return _PyContext_Enter(ts, octx); +} + + +static int +_PyContext_Exit(PyThreadState *ts, PyObject *octx) { ENSURE_Context(octx, -1) PyContext *ctx = (PyContext *)octx; @@ -139,9 +146,6 @@ PyContext_Exit(PyObject *octx) return -1; } - PyThreadState *ts = _PyThreadState_GET(); - assert(ts != NULL); - if (ts->context != (PyObject *)ctx) { /* Can only happen if someone misuses the C API */ PyErr_SetString(PyExc_RuntimeError, @@ -159,6 +163,14 @@ PyContext_Exit(PyObject *octx) return 0; } +int +PyContext_Exit(PyObject *octx) +{ + PyThreadState *ts = _PyThreadState_GET(); + assert(ts != NULL); + return _PyContext_Exit(ts, octx); +} + PyObject * PyContextVar_New(const char *name, PyObject *def) @@ -621,20 +633,22 @@ static PyObject * context_run(PyContext *self, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) { + PyThreadState *ts = _PyThreadState_GET(); + if (nargs < 1) { - PyErr_SetString(PyExc_TypeError, - "run() missing 1 required positional argument"); + _PyErr_SetString(ts, PyExc_TypeError, + "run() missing 1 required positional argument"); return NULL; } - if (PyContext_Enter((PyObject *)self)) { + if (_PyContext_Enter(ts, (PyObject *)self)) { return NULL; } - PyObject *call_result = _PyObject_Vectorcall( - args[0], args + 1, nargs - 1, kwnames); + PyObject *call_result = _PyObject_VectorcallTstate( + ts, args[0], args + 1, nargs - 1, kwnames); - if (PyContext_Exit((PyObject *)self)) { + if (_PyContext_Exit(ts, (PyObject *)self)) { return NULL; } From fc6b1bf869be9fd89c19faf8d12fa473ce5222c8 Mon Sep 17 00:00:00 2001 From: Shu <23287722+susan-shu-c@users.noreply.github.com> Date: Fri, 8 Nov 2019 15:26:35 -0500 Subject: [PATCH 25/34] Clarify amount of dots between package and subpackage (GH-17092) --- Doc/reference/import.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Doc/reference/import.rst b/Doc/reference/import.rst index 0228bfb7e984c5..5cce8ceaa3cb68 100644 --- a/Doc/reference/import.rst +++ b/Doc/reference/import.rst @@ -83,7 +83,7 @@ module. Specifically, any module that contains a ``__path__`` attribute is considered a package. All modules have a name. Subpackage names are separated from their parent -package name by dots, akin to Python's standard attribute access syntax. Thus +package name by a dot, akin to Python's standard attribute access syntax. Thus you might have a module called :mod:`sys` and a package called :mod:`email`, which in turn has a subpackage called :mod:`email.mime` and a module within that subpackage called :mod:`email.mime.text`. From e27449da92b13730a5e11182f329d5da98a5e05b Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Sat, 9 Nov 2019 13:13:36 +0200 Subject: [PATCH 26/34] bpo-38635: Simplify decoding the ZIP64 extra field and make it tolerant to extra data. (GH-16988) --- Lib/zipfile.py | 53 +++++++++++++++----------------------------------- 1 file changed, 16 insertions(+), 37 deletions(-) diff --git a/Lib/zipfile.py b/Lib/zipfile.py index 6504e0eee8b5a1..b0afb9da942b12 100644 --- a/Lib/zipfile.py +++ b/Lib/zipfile.py @@ -465,44 +465,23 @@ def _decodeExtra(self): if ln+4 > len(extra): raise BadZipFile("Corrupt extra field %04x (size=%d)" % (tp, ln)) if tp == 0x0001: - if ln >= 24: - counts = unpack(' Date: Sat, 9 Nov 2019 20:28:31 -0800 Subject: [PATCH 27/34] Minor readability improvement for argument handling in itertools.repeat() (GH-17101) --- Modules/itertoolsmodule.c | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/Modules/itertoolsmodule.c b/Modules/itertoolsmodule.c index 3d39fa2e4737bd..0cb472966d1f95 100644 --- a/Modules/itertoolsmodule.c +++ b/Modules/itertoolsmodule.c @@ -4256,17 +4256,17 @@ repeat_new(PyTypeObject *type, PyObject *args, PyObject *kwds) { repeatobject *ro; PyObject *element; - Py_ssize_t cnt = -1, n_kwds = 0; + Py_ssize_t cnt = -1, n_args; static char *kwargs[] = {"object", "times", NULL}; + n_args = PyTuple_GET_SIZE(args); + if (kwds != NULL) + n_args += PyDict_GET_SIZE(kwds); if (!PyArg_ParseTupleAndKeywords(args, kwds, "O|n:repeat", kwargs, &element, &cnt)) return NULL; - - if (kwds != NULL) - n_kwds = PyDict_GET_SIZE(kwds); /* Does user supply times argument? */ - if ((PyTuple_Size(args) + n_kwds == 2) && cnt < 0) + if (n_args == 2 && cnt < 0) cnt = 0; ro = (repeatobject *)type->tp_alloc(type, 0); From 84ac4376587e35d16b4d0977c4f330d9d04b690a Mon Sep 17 00:00:00 2001 From: Raymond Hettinger Date: Sun, 10 Nov 2019 20:12:04 -0800 Subject: [PATCH 28/34] bpo-38761: Register WeakSet as a MutableSet (GH-17104) --- Lib/test/test_weakset.py | 5 +++++ Lib/weakref.py | 3 +++ .../next/Library/2019-11-10-13-40-33.bpo-38761.P1UUIZ.rst | 1 + 3 files changed, 9 insertions(+) create mode 100644 Misc/NEWS.d/next/Library/2019-11-10-13-40-33.bpo-38761.P1UUIZ.rst diff --git a/Lib/test/test_weakset.py b/Lib/test/test_weakset.py index 569facdd30c11c..49a9b5c3c658a8 100644 --- a/Lib/test/test_weakset.py +++ b/Lib/test/test_weakset.py @@ -2,6 +2,7 @@ from weakref import WeakSet import string from collections import UserString as ustr +from collections.abc import Set, MutableSet import gc import contextlib @@ -437,6 +438,10 @@ def test_len_race(self): def test_repr(self): assert repr(self.s) == repr(self.s.data) + def test_abc(self): + self.assertIsInstance(self.s, Set) + self.assertIsInstance(self.s, MutableSet) + if __name__ == "__main__": unittest.main() diff --git a/Lib/weakref.py b/Lib/weakref.py index d17b3ed38eacb7..e3c2ce2d9b8b86 100644 --- a/Lib/weakref.py +++ b/Lib/weakref.py @@ -33,6 +33,9 @@ "WeakSet", "WeakMethod", "finalize"] +_collections_abc.Set.register(WeakSet) +_collections_abc.MutableSet.register(WeakSet) + class WeakMethod(ref): """ A custom `weakref.ref` subclass which simulates a weak reference to diff --git a/Misc/NEWS.d/next/Library/2019-11-10-13-40-33.bpo-38761.P1UUIZ.rst b/Misc/NEWS.d/next/Library/2019-11-10-13-40-33.bpo-38761.P1UUIZ.rst new file mode 100644 index 00000000000000..4dde5ebc3208ca --- /dev/null +++ b/Misc/NEWS.d/next/Library/2019-11-10-13-40-33.bpo-38761.P1UUIZ.rst @@ -0,0 +1 @@ +WeakSet is now registered as a collections.abc.MutableSet. From a0ed99bca8475cbc82e9202aa354faba2a4620f4 Mon Sep 17 00:00:00 2001 From: Brandt Bucher Date: Mon, 11 Nov 2019 12:47:48 -0800 Subject: [PATCH 29/34] bpo-38438: Simplify argparse "star nargs" usage. (GH-17106) --- Doc/library/argparse.rst | 2 +- Doc/library/pdb.rst | 6 +++--- Lib/argparse.py | 6 +++++- Lib/pydoc_data/topics.py | 6 +++--- Lib/test/test_argparse.py | 14 +++++++------- .../2019-11-11-06-14-25.bpo-38438.vSVeHN.rst | 1 + 6 files changed, 20 insertions(+), 15 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2019-11-11-06-14-25.bpo-38438.vSVeHN.rst diff --git a/Doc/library/argparse.rst b/Doc/library/argparse.rst index 818acf484cdf39..9a1cf3a53c4427 100644 --- a/Doc/library/argparse.rst +++ b/Doc/library/argparse.rst @@ -449,7 +449,7 @@ default values to each of the argument help messages:: >>> parser.add_argument('--foo', type=int, default=42, help='FOO!') >>> parser.add_argument('bar', nargs='*', default=[1, 2, 3], help='BAR!') >>> parser.print_help() - usage: PROG [-h] [--foo FOO] [bar [bar ...]] + usage: PROG [-h] [--foo FOO] [bar ...] positional arguments: bar BAR! (default: [1, 2, 3]) diff --git a/Doc/library/pdb.rst b/Doc/library/pdb.rst index f26b6a8b553b36..0b9a687b14e3a3 100644 --- a/Doc/library/pdb.rst +++ b/Doc/library/pdb.rst @@ -296,20 +296,20 @@ by the local file. Temporary breakpoint, which is removed automatically when it is first hit. The arguments are the same as for :pdbcmd:`break`. -.. pdbcommand:: cl(ear) [filename:lineno | bpnumber [bpnumber ...]] +.. pdbcommand:: cl(ear) [filename:lineno | bpnumber ...] With a *filename:lineno* argument, clear all the breakpoints at this line. With a space separated list of breakpoint numbers, clear those breakpoints. Without argument, clear all breaks (but first ask confirmation). -.. pdbcommand:: disable [bpnumber [bpnumber ...]] +.. pdbcommand:: disable [bpnumber ...] Disable the breakpoints given as a space separated list of breakpoint numbers. Disabling a breakpoint means it cannot cause the program to stop execution, but unlike clearing a breakpoint, it remains in the list of breakpoints and can be (re-)enabled. -.. pdbcommand:: enable [bpnumber [bpnumber ...]] +.. pdbcommand:: enable [bpnumber ...] Enable the breakpoints specified. diff --git a/Lib/argparse.py b/Lib/argparse.py index 13af7ac2392174..94e1b8ad0ed18e 100644 --- a/Lib/argparse.py +++ b/Lib/argparse.py @@ -591,7 +591,11 @@ def _format_args(self, action, default_metavar): elif action.nargs == OPTIONAL: result = '[%s]' % get_metavar(1) elif action.nargs == ZERO_OR_MORE: - result = '[%s [%s ...]]' % get_metavar(2) + metavar = get_metavar(1) + if len(metavar) == 2: + result = '[%s [%s ...]]' % metavar + else: + result = '[%s ...]' % metavar elif action.nargs == ONE_OR_MORE: result = '%s [%s ...]' % get_metavar(2) elif action.nargs == REMAINDER: diff --git a/Lib/pydoc_data/topics.py b/Lib/pydoc_data/topics.py index 1fb19f241ac31c..d3559e445ab013 100644 --- a/Lib/pydoc_data/topics.py +++ b/Lib/pydoc_data/topics.py @@ -3903,7 +3903,7 @@ 'is\n' ' first hit. The arguments are the same as for "break".\n' '\n' - 'cl(ear) [filename:lineno | bpnumber [bpnumber ...]]\n' + 'cl(ear) [filename:lineno | bpnumber ...]\n' '\n' ' With a *filename:lineno* argument, clear all the breakpoints ' 'at\n' @@ -3913,7 +3913,7 @@ 'first\n' ' ask confirmation).\n' '\n' - 'disable [bpnumber [bpnumber ...]]\n' + 'disable [bpnumber ...]\n' '\n' ' Disable the breakpoints given as a space separated list of\n' ' breakpoint numbers. Disabling a breakpoint means it cannot ' @@ -3922,7 +3922,7 @@ 'breakpoint, it\n' ' remains in the list of breakpoints and can be (re-)enabled.\n' '\n' - 'enable [bpnumber [bpnumber ...]]\n' + 'enable [bpnumber ...]\n' '\n' ' Enable the breakpoints specified.\n' '\n' diff --git a/Lib/test/test_argparse.py b/Lib/test/test_argparse.py index a97c921852c7bf..60bf19918b79e5 100644 --- a/Lib/test/test_argparse.py +++ b/Lib/test/test_argparse.py @@ -2163,7 +2163,7 @@ def test_subparser1_help(self): def test_subparser2_help(self): self._test_subparser_help('5.0 2 -h', textwrap.dedent('''\ - usage: PROG bar 2 [-h] [-y {1,2,3}] [z [z ...]] + usage: PROG bar 2 [-h] [-y {1,2,3}] [z ...] 2 description @@ -2697,10 +2697,10 @@ def get_parser(self, required): ] usage_when_not_required = '''\ - usage: PROG [-h] [--foo | --spam SPAM | badger [badger ...]] + usage: PROG [-h] [--foo | --spam SPAM | badger ...] ''' usage_when_required = '''\ - usage: PROG [-h] (--foo | --spam SPAM | badger [badger ...]) + usage: PROG [-h] (--foo | --spam SPAM | badger ...) ''' help = '''\ @@ -3494,11 +3494,11 @@ class TestHelpUsage(HelpTestCase): ]) ] usage = '''\ - usage: PROG [-h] [-w W [W ...]] [-x [X [X ...]]] [--foo | --no-foo] + usage: PROG [-h] [-w W [W ...]] [-x [X ...]] [--foo | --no-foo] [--bar | --no-bar] [-f | --foobar | --no-foobar | --barfoo | --no-barfoo] [-y [Y]] [-z Z Z Z] - a b b [c] [d [d ...]] e [e ...] + a b b [c] [d ...] e [e ...] ''' help = usage + '''\ @@ -3510,7 +3510,7 @@ class TestHelpUsage(HelpTestCase): optional arguments: -h, --help show this help message and exit -w W [W ...] w - -x [X [X ...]] x + -x [X ...] x --foo, --no-foo Whether to foo --bar, --no-bar Whether to bar (default: True) -f, --foobar, --no-foobar, --barfoo, --no-barfoo @@ -5113,7 +5113,7 @@ def test_nargs_zeroormore_metavar_length0(self): self.do_test_exception(nargs="*", metavar=tuple()) def test_nargs_zeroormore_metavar_length1(self): - self.do_test_exception(nargs="*", metavar=("1",)) + self.do_test_no_exception(nargs="*", metavar=("1",)) def test_nargs_zeroormore_metavar_length2(self): self.do_test_no_exception(nargs="*", metavar=("1", "2")) diff --git a/Misc/NEWS.d/next/Library/2019-11-11-06-14-25.bpo-38438.vSVeHN.rst b/Misc/NEWS.d/next/Library/2019-11-11-06-14-25.bpo-38438.vSVeHN.rst new file mode 100644 index 00000000000000..69ea2de02f73a8 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2019-11-11-06-14-25.bpo-38438.vSVeHN.rst @@ -0,0 +1 @@ +Simplify the :mod:`argparse` usage message for ``nargs="*"``. \ No newline at end of file From 98480cef9dba04794bd61c7e7cca643d384c8c35 Mon Sep 17 00:00:00 2001 From: Jonathan Scholbach Date: Tue, 12 Nov 2019 01:49:41 +0100 Subject: [PATCH 30/34] bpo-38771: Explict test for None in code example (GH-17108) --- Doc/library/collections.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Doc/library/collections.rst b/Doc/library/collections.rst index 90a3f4bea9a45b..a5e8d04099b22f 100644 --- a/Doc/library/collections.rst +++ b/Doc/library/collections.rst @@ -162,7 +162,7 @@ environment variables which in turn take precedence over default values:: parser.add_argument('-u', '--user') parser.add_argument('-c', '--color') namespace = parser.parse_args() - command_line_args = {k:v for k, v in vars(namespace).items() if v} + command_line_args = {k: v for k, v in vars(namespace).items() if v is not None} combined = ChainMap(command_line_args, os.environ, defaults) print(combined['color']) From 051ff526b5dc2c40c4a53d87089740358822edfa Mon Sep 17 00:00:00 2001 From: Manjusaka Date: Tue, 12 Nov 2019 15:30:18 +0800 Subject: [PATCH 31/34] bpo-38565: add new cache_parameters method for lru_cache (GH-16916) --- Doc/library/functools.rst | 8 ++++++++ Lib/functools.py | 2 ++ Lib/test/test_functools.py | 11 +++++++++++ .../Library/2019-10-24-08-10-30.bpo-38565.SWSUst.rst | 1 + 4 files changed, 22 insertions(+) create mode 100644 Misc/NEWS.d/next/Library/2019-10-24-08-10-30.bpo-38565.SWSUst.rst diff --git a/Doc/library/functools.rst b/Doc/library/functools.rst index d3debac8432b19..cedc3ad5ec0a61 100644 --- a/Doc/library/functools.rst +++ b/Doc/library/functools.rst @@ -108,6 +108,11 @@ The :mod:`functools` module defines the following functions: cached separately. For example, ``f(3)`` and ``f(3.0)`` will be treated as distinct calls with distinct results. + The wrapped function is instrumented with a :func:`cache_parameters` + function that returns a new :class:`dict` showing the values for *maxsize* + and *typed*. This is for information purposes only. Mutating the values + has no effect. + To help measure the effectiveness of the cache and tune the *maxsize* parameter, the wrapped function is instrumented with a :func:`cache_info` function that returns a :term:`named tuple` showing *hits*, *misses*, @@ -178,6 +183,9 @@ The :mod:`functools` module defines the following functions: .. versionchanged:: 3.8 Added the *user_function* option. + .. versionadded:: 3.9 + Added the function :func:`cache_parameters` + .. decorator:: total_ordering Given a class defining one or more rich comparison ordering methods, this diff --git a/Lib/functools.py b/Lib/functools.py index 3192bd02d93e56..2c01b2e59524bf 100644 --- a/Lib/functools.py +++ b/Lib/functools.py @@ -499,6 +499,7 @@ def lru_cache(maxsize=128, typed=False): # The user_function was passed in directly via the maxsize argument user_function, maxsize = maxsize, 128 wrapper = _lru_cache_wrapper(user_function, maxsize, typed, _CacheInfo) + wrapper.cache_parameters = lambda : {'maxsize': maxsize, 'typed': typed} return update_wrapper(wrapper, user_function) elif maxsize is not None: raise TypeError( @@ -506,6 +507,7 @@ def lru_cache(maxsize=128, typed=False): def decorating_function(user_function): wrapper = _lru_cache_wrapper(user_function, maxsize, typed, _CacheInfo) + wrapper.cache_parameters = lambda : {'maxsize': maxsize, 'typed': typed} return update_wrapper(wrapper, user_function) return decorating_function diff --git a/Lib/test/test_functools.py b/Lib/test/test_functools.py index c300270d49e5ec..a97ca398e77a33 100644 --- a/Lib/test/test_functools.py +++ b/Lib/test/test_functools.py @@ -1655,6 +1655,17 @@ def orig(x, y): f_copy = copy.deepcopy(f) self.assertIs(f_copy, f) + def test_lru_cache_parameters(self): + @self.module.lru_cache(maxsize=2) + def f(): + return 1 + self.assertEqual(f.cache_parameters(), {'maxsize': 2, "typed": False}) + + @self.module.lru_cache(maxsize=1000, typed=True) + def f(): + return 1 + self.assertEqual(f.cache_parameters(), {'maxsize': 1000, "typed": True}) + @py_functools.lru_cache() def py_cached_func(x, y): diff --git a/Misc/NEWS.d/next/Library/2019-10-24-08-10-30.bpo-38565.SWSUst.rst b/Misc/NEWS.d/next/Library/2019-10-24-08-10-30.bpo-38565.SWSUst.rst new file mode 100644 index 00000000000000..34d7afcf36df62 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2019-10-24-08-10-30.bpo-38565.SWSUst.rst @@ -0,0 +1 @@ +Add new cache_parameters() method for functools.lru_cache() to better support pickling. \ No newline at end of file From 733b9a308e3c49855888e2e12397ae56d831e780 Mon Sep 17 00:00:00 2001 From: Raymond Hettinger Date: Mon, 11 Nov 2019 23:35:06 -0800 Subject: [PATCH 32/34] bpo-38385: Fix iterator/iterable terminology in statistics docs (GH-17111) --- Doc/library/statistics.rst | 18 +++++++++--------- Lib/statistics.py | 2 +- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/Doc/library/statistics.rst b/Doc/library/statistics.rst index 00c0b539edce95..a790ed81a5607e 100644 --- a/Doc/library/statistics.rst +++ b/Doc/library/statistics.rst @@ -77,7 +77,7 @@ However, for reading convenience, most of the examples show sorted sequences. .. function:: mean(data) - Return the sample arithmetic mean of *data* which can be a sequence or iterator. + Return the sample arithmetic mean of *data* which can be a sequence or iterable. The arithmetic mean is the sum of the data divided by the number of data points. It is commonly called "the average", although it is only one of many @@ -122,7 +122,7 @@ However, for reading convenience, most of the examples show sorted sequences. Convert *data* to floats and compute the arithmetic mean. This runs faster than the :func:`mean` function and it always returns a - :class:`float`. The *data* may be a sequence or iterator. If the input + :class:`float`. The *data* may be a sequence or iterable. If the input dataset is empty, raises a :exc:`StatisticsError`. .. doctest:: @@ -143,7 +143,7 @@ However, for reading convenience, most of the examples show sorted sequences. Raises a :exc:`StatisticsError` if the input dataset is empty, if it contains a zero, or if it contains a negative value. - The *data* may be a sequence or iterator. + The *data* may be a sequence or iterable. No special efforts are made to achieve exact results. (However, this may change in the future.) @@ -158,7 +158,7 @@ However, for reading convenience, most of the examples show sorted sequences. .. function:: harmonic_mean(data) - Return the harmonic mean of *data*, a sequence or iterator of + Return the harmonic mean of *data*, a sequence or iterable of real-valued numbers. The harmonic mean, sometimes called the subcontrary mean, is the @@ -202,7 +202,7 @@ However, for reading convenience, most of the examples show sorted sequences. Return the median (middle value) of numeric data, using the common "mean of middle two" method. If *data* is empty, :exc:`StatisticsError` is raised. - *data* can be a sequence or iterator. + *data* can be a sequence or iterable. The median is a robust measure of central location and is less affected by the presence of outliers. When the number of data points is odd, the @@ -231,7 +231,7 @@ However, for reading convenience, most of the examples show sorted sequences. .. function:: median_low(data) Return the low median of numeric data. If *data* is empty, - :exc:`StatisticsError` is raised. *data* can be a sequence or iterator. + :exc:`StatisticsError` is raised. *data* can be a sequence or iterable. The low median is always a member of the data set. When the number of data points is odd, the middle value is returned. When it is even, the smaller of @@ -251,7 +251,7 @@ However, for reading convenience, most of the examples show sorted sequences. .. function:: median_high(data) Return the high median of data. If *data* is empty, :exc:`StatisticsError` - is raised. *data* can be a sequence or iterator. + is raised. *data* can be a sequence or iterable. The high median is always a member of the data set. When the number of data points is odd, the middle value is returned. When it is even, the larger of @@ -272,7 +272,7 @@ However, for reading convenience, most of the examples show sorted sequences. Return the median of grouped continuous data, calculated as the 50th percentile, using interpolation. If *data* is empty, :exc:`StatisticsError` - is raised. *data* can be a sequence or iterator. + is raised. *data* can be a sequence or iterable. .. doctest:: @@ -381,7 +381,7 @@ However, for reading convenience, most of the examples show sorted sequences. .. function:: pvariance(data, mu=None) - Return the population variance of *data*, a non-empty sequence or iterator + Return the population variance of *data*, a non-empty sequence or iterable of real-valued numbers. Variance, or second moment about the mean, is a measure of the variability (spread or dispersion) of data. A large variance indicates that the data is spread out; a small variance indicates diff --git a/Lib/statistics.py b/Lib/statistics.py index 461ffae3f4914e..1e95c0b6639f17 100644 --- a/Lib/statistics.py +++ b/Lib/statistics.py @@ -744,7 +744,7 @@ def variance(data, xbar=None): def pvariance(data, mu=None): """Return the population variance of ``data``. - data should be a sequence or iterator of Real-valued numbers, with at least one + data should be a sequence or iterable of Real-valued numbers, with at least one value. The optional argument mu, if given, should be the mean of the data. If it is missing or None, the mean is automatically calculated. From 1b5aa07a1799c5f9ac474ba5695101b5b1b57866 Mon Sep 17 00:00:00 2001 From: Callum Date: Mon, 4 Nov 2019 11:49:59 +0000 Subject: [PATCH 33/34] Add is_reading() method to _UnixReadPipeTransport Unix read pipe is missing method available for other read transports within the aynscio protocol, which checks that the transport read is not paused or the transport is not closing. --- Lib/asyncio/unix_events.py | 3 +++ Lib/test/test_asyncio/test_unix_events.py | 14 ++++++++++++++ 2 files changed, 17 insertions(+) diff --git a/Lib/asyncio/unix_events.py b/Lib/asyncio/unix_events.py index d8f653045aee4c..10946f13e072a7 100644 --- a/Lib/asyncio/unix_events.py +++ b/Lib/asyncio/unix_events.py @@ -522,6 +522,9 @@ def resume_reading(self): if self._loop.get_debug(): logger.debug("%r resumes reading", self) + def is_reading(self): + return not self._paused and not self._closing + def set_protocol(self, protocol): self._protocol = protocol diff --git a/Lib/test/test_asyncio/test_unix_events.py b/Lib/test/test_asyncio/test_unix_events.py index 5487b7afef8326..7037b2091af88d 100644 --- a/Lib/test/test_asyncio/test_unix_events.py +++ b/Lib/test/test_asyncio/test_unix_events.py @@ -740,6 +740,20 @@ def test_resume_reading(self, m_read): tr.resume_reading() self.loop.assert_reader(5, tr._read_ready) + @mock.patch('os.read') + def test_is_reading(self, m_read): + tr = self.read_pipe_transport() + tr._paused = False + tr._closing = False + self.assertTrue(tr.is_reading()) + tr._paused = True + self.assertFalse(tr.is_reading()) + tr._paused = False + tr._closing = True + self.assertFalse(tr.is_reading()) + tr._closing = False + self.assertTrue(tr.is_reading()) + @mock.patch('os.read') def test_close(self, m_read): tr = self.read_pipe_transport() From 1dbad62d285f5d7cb4115d810d3d247f17127886 Mon Sep 17 00:00:00 2001 From: Callum Date: Tue, 12 Nov 2019 16:21:10 +0000 Subject: [PATCH 34/34] Add new test case file for UNIX pipe transport functional tests --- Lib/test/test_asyncio/test_unix_pipes.py | 103 +++++++++++++++++++++++ 1 file changed, 103 insertions(+) create mode 100644 Lib/test/test_asyncio/test_unix_pipes.py diff --git a/Lib/test/test_asyncio/test_unix_pipes.py b/Lib/test/test_asyncio/test_unix_pipes.py new file mode 100644 index 00000000000000..c3826d5c69c38b --- /dev/null +++ b/Lib/test/test_asyncio/test_unix_pipes.py @@ -0,0 +1,103 @@ +""" +Functional tests for Unix transport pipes, designed around opening two sockets +on the localhost and sending information between +""" + +import unittest +import sys +import os +import tempfile +import threading + +if sys.platform == 'win32': + raise unittest.SkipTest('UNIX only') + +import asyncio +from asyncio import unix_events + + +def tearDownModule(): + asyncio.set_event_loop_policy(None) + + +class UnixReadPipeTransportFuncTests(unittest.TestCase): + """ + Verify that transports on Unix can facilitate reading and have access to + all state methods: verify using class internals + """ + + async def register_read_handle(self): + """ + Wait for read_handle and then register it to the loop + """ + self.transport, self.protocol = await self.loop.connect_read_pipe( + asyncio.BaseProtocol, + self.read_handle, + ) + + def setup_read_handle(self): + """ + Open the read handle and record it in an attribute + """ + self.read_handle = open(self.pipe, "r") + + def setup_write_handle(self): + """ + Open the write handle and record it in an attribute + """ + self.write_handle = open(self.pipe, "w") + + def setUp(self): + """ + Create the UNIX pipe and register the read end to the loop, and connect + a write handle asynchronously + """ + self.loop = asyncio.get_event_loop() + self.temp_dir = tempfile.TemporaryDirectory(suffix="async_unix_events") + self.pipe = os.path.join(self.temp_dir.name, "unix_pipe") + os.mkfifo(self.pipe) + + # Set the threads to open the handles going + r_handle_thread = threading.Thread(target=self.setup_read_handle) + r_handle_thread.start() + w_handle_thread = threading.Thread(target=self.setup_write_handle) + w_handle_thread.start() + + # Wait for pipe pair to connect + r_handle_thread.join() + w_handle_thread.join() + + # Once pipe is connected, get the read transport + self.loop.run_until_complete(self.register_read_handle()) + + self.assertIsInstance(self.transport, + unix_events._UnixReadPipeTransport) + + def tearDown(self): + """ + Destroy the read transport and the pipe + """ + self.transport.close() + self.write_handle.close() + self.read_handle.close() + self.loop._run_once() + os.unlink(self.pipe) + self.temp_dir.cleanup() + self.loop.close() + + def test_is_reading(self): + """ + Verify that is_reading returns True unless transport is closed/closing + or paused + """ + self.assertTrue(self.transport.is_reading()) + self.transport.pause_reading() + self.assertFalse(self.transport.is_reading()) + self.transport.resume_reading() + self.assertTrue(self.transport.is_reading()) + self.transport.close() + self.assertFalse(self.transport.is_reading()) + + +if __name__ == "__main__": + unittest.main()