From a92719dfee57f0b7cc22c4da7362cf20af8dc992 Mon Sep 17 00:00:00 2001 From: Jeffrey Bosboom Date: Fri, 5 Sep 2025 01:17:23 -0700 Subject: [PATCH 1/7] Optimize fill_time for typical timestamps While file timestamps can be anything the file system can store, most lie between the recent past and the near future. Optimize fill_time for typical timestamps in three ways: - When possible, convert to nanoseconds with C arithmetic. - When using C arithmetic and the seconds member is not required (for st_birthtime), avoid creating a long object. - When using C arithmetic, reorder the code to avoid the null checks implied in Py_XDECREF. --- Modules/posixmodule.c | 46 ++++++++++++++++++++++++++++++++----------- 1 file changed, 34 insertions(+), 12 deletions(-) diff --git a/Modules/posixmodule.c b/Modules/posixmodule.c index f7e721e52e11b0..2e7e2c43174cbe 100644 --- a/Modules/posixmodule.c +++ b/Modules/posixmodule.c @@ -2588,12 +2588,43 @@ static int fill_time(PyObject *module, PyObject *v, int s_index, int f_index, int ns_index, time_t sec, unsigned long nsec) { assert(!PyErr_Occurred()); +#define S_TO_NS (1000000000LL) + assert(nsec < S_TO_NS); + + if (f_index >= 0) { + PyObject *float_s = PyFloat_FromDouble(sec + 1e-9*nsec); + if (!float_s) { + return -1; + } + PyStructSequence_SET_ITEM(v, f_index, float_s); + } + + /* 1677-09-21 00:12:44 to 2262-04-11 23:47:15 UTC inclusive */ + if ((LLONG_MIN / S_TO_NS) <= sec && sec <= (LONG_MAX / S_TO_NS - 1)) { + if (s_index >= 0) { + PyObject *s = _PyLong_FromTime_t(sec); + if (!s) { + return -1; + } + PyStructSequence_SET_ITEM(v, s_index, s); + } + + if (ns_index >= 0) { + PyObject *ns_total = PyLong_FromLongLong(sec * S_TO_NS + nsec); + if (!ns_total) { + return -1; + } + PyStructSequence_SET_ITEM(v, ns_index, ns_total); + } + + assert(!PyErr_Occurred()); + return 0; + } +#undef S_TO_NS int res = -1; PyObject *s_in_ns = NULL; PyObject *ns_total = NULL; - PyObject *float_s = NULL; - PyObject *s = _PyLong_FromTime_t(sec); PyObject *ns_fractional = PyLong_FromUnsignedLong(nsec); if (!(s && ns_fractional)) { @@ -2606,11 +2637,7 @@ fill_time(PyObject *module, PyObject *v, int s_index, int f_index, int ns_index, } ns_total = PyNumber_Add(s_in_ns, ns_fractional); - if (!ns_total) - goto exit; - - float_s = PyFloat_FromDouble(sec + 1e-9*nsec); - if (!float_s) { + if (!ns_total) { goto exit; } @@ -2618,10 +2645,6 @@ fill_time(PyObject *module, PyObject *v, int s_index, int f_index, int ns_index, PyStructSequence_SET_ITEM(v, s_index, s); s = NULL; } - if (f_index >= 0) { - PyStructSequence_SET_ITEM(v, f_index, float_s); - float_s = NULL; - } if (ns_index >= 0) { PyStructSequence_SET_ITEM(v, ns_index, ns_total); ns_total = NULL; @@ -2635,7 +2658,6 @@ fill_time(PyObject *module, PyObject *v, int s_index, int f_index, int ns_index, Py_XDECREF(ns_fractional); Py_XDECREF(s_in_ns); Py_XDECREF(ns_total); - Py_XDECREF(float_s); return res; } From 6f05f251d534a39efbad5d213f7155c7ea36d081 Mon Sep 17 00:00:00 2001 From: Jeffrey Bosboom Date: Fri, 5 Sep 2025 16:17:44 -0700 Subject: [PATCH 2/7] Address review comments --- Modules/posixmodule.c | 86 +++++++++++++++++++------------------------ 1 file changed, 38 insertions(+), 48 deletions(-) diff --git a/Modules/posixmodule.c b/Modules/posixmodule.c index 2e7e2c43174cbe..49acb39dcbe4e2 100644 --- a/Modules/posixmodule.c +++ b/Modules/posixmodule.c @@ -2591,74 +2591,64 @@ fill_time(PyObject *module, PyObject *v, int s_index, int f_index, int ns_index, #define S_TO_NS (1000000000LL) assert(nsec < S_TO_NS); + if (s_index >= 0) { + PyObject *s = _PyLong_FromTime_t(sec); + if (!s) { + return -1; + } + PyStructSequence_SET_ITEM(v, s_index, s); + } + if (f_index >= 0) { - PyObject *float_s = PyFloat_FromDouble(sec + 1e-9*nsec); + PyObject *float_s = PyFloat_FromDouble(sec + 1e-9 * nsec); if (!float_s) { return -1; } PyStructSequence_SET_ITEM(v, f_index, float_s); } - /* 1677-09-21 00:12:44 to 2262-04-11 23:47:15 UTC inclusive */ - if ((LLONG_MIN / S_TO_NS) <= sec && sec <= (LONG_MAX / S_TO_NS - 1)) { - if (s_index >= 0) { - PyObject *s = _PyLong_FromTime_t(sec); - if (!s) { - return -1; - } - PyStructSequence_SET_ITEM(v, s_index, s); - } - - if (ns_index >= 0) { + int res = -1; + if (ns_index >= 0) { + /* 1677-09-21 00:12:44 to 2262-04-11 23:47:15 UTC inclusive */ + if ((LLONG_MIN / S_TO_NS) <= sec && sec <= (LONG_MAX / S_TO_NS - 1)) { PyObject *ns_total = PyLong_FromLongLong(sec * S_TO_NS + nsec); if (!ns_total) { return -1; } PyStructSequence_SET_ITEM(v, ns_index, ns_total); + res = 0; } + else { + PyObject *s_in_ns = NULL; + PyObject *ns_total = NULL; + PyObject *s = _PyLong_FromTime_t(sec); + PyObject *ns_fractional = PyLong_FromUnsignedLong(nsec); + if (!(s && ns_fractional)) { + goto exit; + } - assert(!PyErr_Occurred()); - return 0; - } -#undef S_TO_NS - - int res = -1; - PyObject *s_in_ns = NULL; - PyObject *ns_total = NULL; - PyObject *s = _PyLong_FromTime_t(sec); - PyObject *ns_fractional = PyLong_FromUnsignedLong(nsec); - if (!(s && ns_fractional)) { - goto exit; - } - - s_in_ns = PyNumber_Multiply(s, get_posix_state(module)->billion); - if (!s_in_ns) { - goto exit; - } + s_in_ns = PyNumber_Multiply(s, get_posix_state(module)->billion); + if (!s_in_ns) { + goto exit; + } - ns_total = PyNumber_Add(s_in_ns, ns_fractional); - if (!ns_total) { - goto exit; - } + ns_total = PyNumber_Add(s_in_ns, ns_fractional); + if (!ns_total) { + goto exit; + } + PyStructSequence_SET_ITEM(v, ns_index, ns_total); + res = 0; - if (s_index >= 0) { - PyStructSequence_SET_ITEM(v, s_index, s); - s = NULL; - } - if (ns_index >= 0) { - PyStructSequence_SET_ITEM(v, ns_index, ns_total); - ns_total = NULL; + exit: + Py_XDECREF(s); + Py_XDECREF(ns_fractional); + Py_XDECREF(s_in_ns); + } } assert(!PyErr_Occurred()); - res = 0; - -exit: - Py_XDECREF(s); - Py_XDECREF(ns_fractional); - Py_XDECREF(s_in_ns); - Py_XDECREF(ns_total); return res; + #undef S_TO_NS } #ifdef MS_WINDOWS From bbab367c2734e1238618008b994bd7881f8ddeeb Mon Sep 17 00:00:00 2001 From: Jeffrey Bosboom Date: Fri, 5 Sep 2025 18:09:56 -0700 Subject: [PATCH 3/7] assert(!PyErr_Occurred()) only on success --- Modules/posixmodule.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Modules/posixmodule.c b/Modules/posixmodule.c index 49acb39dcbe4e2..a63bc94d91ad09 100644 --- a/Modules/posixmodule.c +++ b/Modules/posixmodule.c @@ -2616,6 +2616,7 @@ fill_time(PyObject *module, PyObject *v, int s_index, int f_index, int ns_index, return -1; } PyStructSequence_SET_ITEM(v, ns_index, ns_total); + assert(!PyErr_Occurred()); res = 0; } else { @@ -2637,6 +2638,7 @@ fill_time(PyObject *module, PyObject *v, int s_index, int f_index, int ns_index, goto exit; } PyStructSequence_SET_ITEM(v, ns_index, ns_total); + assert(!PyErr_Occurred()); res = 0; exit: @@ -2646,7 +2648,6 @@ fill_time(PyObject *module, PyObject *v, int s_index, int f_index, int ns_index, } } - assert(!PyErr_Occurred()); return res; #undef S_TO_NS } From 1a9183c390805310566424125f882e2acf2b3a32 Mon Sep 17 00:00:00 2001 From: Jeffrey Bosboom Date: Sat, 6 Sep 2025 20:03:58 -0700 Subject: [PATCH 4/7] Fix typo in fast path upper bound --- Modules/posixmodule.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Modules/posixmodule.c b/Modules/posixmodule.c index a63bc94d91ad09..fda13ad9ac900c 100644 --- a/Modules/posixmodule.c +++ b/Modules/posixmodule.c @@ -2610,7 +2610,7 @@ fill_time(PyObject *module, PyObject *v, int s_index, int f_index, int ns_index, int res = -1; if (ns_index >= 0) { /* 1677-09-21 00:12:44 to 2262-04-11 23:47:15 UTC inclusive */ - if ((LLONG_MIN / S_TO_NS) <= sec && sec <= (LONG_MAX / S_TO_NS - 1)) { + if ((LLONG_MIN / S_TO_NS) <= sec && sec <= (LLONG_MAX / S_TO_NS - 1)) { PyObject *ns_total = PyLong_FromLongLong(sec * S_TO_NS + nsec); if (!ns_total) { return -1; From 2c74007be5dab4a615ce5a3543d9fcd87916c747 Mon Sep 17 00:00:00 2001 From: Jeffrey Bosboom Date: Sat, 6 Sep 2025 20:04:44 -0700 Subject: [PATCH 5/7] Add tests for boundaries of fast path --- Lib/test/test_os.py | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/Lib/test/test_os.py b/Lib/test/test_os.py index 9827a7f12ea21d..e90e665d5a4f9f 100644 --- a/Lib/test/test_os.py +++ b/Lib/test/test_os.py @@ -1064,9 +1064,14 @@ def test_large_time(self): if self.get_file_system(self.dirname) != "NTFS": self.skipTest("requires NTFS") - large = 5000000000 # some day in 2128 - os.utime(self.fname, (large, large)) - self.assertEqual(os.stat(self.fname).st_mtime, large) + times = ( + 5000000000, # some day in 2128 + # boundaries of the fast path cutoff in posixmodule.c:fill_time + -9223372037, -9223372036, 9223372035, 9223372036, + ) + for large in times: + os.utime(self.fname, (large, large)) + self.assertEqual(os.stat(self.fname).st_mtime, large) def test_utime_invalid_arguments(self): # seconds and nanoseconds parameters are mutually exclusive From 30d790f6e41b335d5a98e649cdd38c4ccfdb32a7 Mon Sep 17 00:00:00 2001 From: Jeffrey Bosboom Date: Sat, 6 Sep 2025 20:15:04 -0700 Subject: [PATCH 6/7] Add blurb --- .../next/Library/2025-09-06-20-09-32.gh-issue-138535.mlntEe.rst | 2 ++ 1 file changed, 2 insertions(+) create mode 100644 Misc/NEWS.d/next/Library/2025-09-06-20-09-32.gh-issue-138535.mlntEe.rst diff --git a/Misc/NEWS.d/next/Library/2025-09-06-20-09-32.gh-issue-138535.mlntEe.rst b/Misc/NEWS.d/next/Library/2025-09-06-20-09-32.gh-issue-138535.mlntEe.rst new file mode 100644 index 00000000000000..3fa8f48d56a2bd --- /dev/null +++ b/Misc/NEWS.d/next/Library/2025-09-06-20-09-32.gh-issue-138535.mlntEe.rst @@ -0,0 +1,2 @@ +Speed up :func:`os.stat` for files with reasonable timestamps. Contributed +by Jeffrey Bosboom. From be7d76f3e85b17f708eb48c8f0b94d9439312077 Mon Sep 17 00:00:00 2001 From: Jeffrey Bosboom Date: Mon, 8 Sep 2025 14:03:22 -0700 Subject: [PATCH 7/7] Apply suggestions from code review Co-authored-by: Victor Stinner --- Lib/test/test_os.py | 5 +++-- Modules/posixmodule.c | 24 ++++++++++++------------ 2 files changed, 15 insertions(+), 14 deletions(-) diff --git a/Lib/test/test_os.py b/Lib/test/test_os.py index e90e665d5a4f9f..fd32b6990ec36a 100644 --- a/Lib/test/test_os.py +++ b/Lib/test/test_os.py @@ -1070,8 +1070,9 @@ def test_large_time(self): -9223372037, -9223372036, 9223372035, 9223372036, ) for large in times: - os.utime(self.fname, (large, large)) - self.assertEqual(os.stat(self.fname).st_mtime, large) + with self.subTest(large=large): + os.utime(self.fname, (large, large)) + self.assertEqual(os.stat(self.fname).st_mtime, large) def test_utime_invalid_arguments(self): # seconds and nanoseconds parameters are mutually exclusive diff --git a/Modules/posixmodule.c b/Modules/posixmodule.c index fda13ad9ac900c..fccffe059d2ee3 100644 --- a/Modules/posixmodule.c +++ b/Modules/posixmodule.c @@ -2588,20 +2588,20 @@ static int fill_time(PyObject *module, PyObject *v, int s_index, int f_index, int ns_index, time_t sec, unsigned long nsec) { assert(!PyErr_Occurred()); -#define S_TO_NS (1000000000LL) - assert(nsec < S_TO_NS); +#define SEC_TO_NS (1000000000LL) + assert(nsec < SEC_TO_NS); if (s_index >= 0) { PyObject *s = _PyLong_FromTime_t(sec); - if (!s) { + if (s == NULL) { return -1; } PyStructSequence_SET_ITEM(v, s_index, s); } if (f_index >= 0) { - PyObject *float_s = PyFloat_FromDouble(sec + 1e-9 * nsec); - if (!float_s) { + PyObject *float_s = PyFloat_FromDouble((double)sec + 1e-9 * nsec); + if (float_s == NULL) { return -1; } PyStructSequence_SET_ITEM(v, f_index, float_s); @@ -2610,9 +2610,9 @@ fill_time(PyObject *module, PyObject *v, int s_index, int f_index, int ns_index, int res = -1; if (ns_index >= 0) { /* 1677-09-21 00:12:44 to 2262-04-11 23:47:15 UTC inclusive */ - if ((LLONG_MIN / S_TO_NS) <= sec && sec <= (LLONG_MAX / S_TO_NS - 1)) { - PyObject *ns_total = PyLong_FromLongLong(sec * S_TO_NS + nsec); - if (!ns_total) { + if ((LLONG_MIN/SEC_TO_NS) <= sec && sec <= (LLONG_MAX/SEC_TO_NS - 1)) { + PyObject *ns_total = PyLong_FromLongLong(sec * SEC_TO_NS + nsec); + if (ns_total == NULL) { return -1; } PyStructSequence_SET_ITEM(v, ns_index, ns_total); @@ -2624,17 +2624,17 @@ fill_time(PyObject *module, PyObject *v, int s_index, int f_index, int ns_index, PyObject *ns_total = NULL; PyObject *s = _PyLong_FromTime_t(sec); PyObject *ns_fractional = PyLong_FromUnsignedLong(nsec); - if (!(s && ns_fractional)) { + if (s == NULL || ns_fractional == NULL) { goto exit; } s_in_ns = PyNumber_Multiply(s, get_posix_state(module)->billion); - if (!s_in_ns) { + if (s_in_ns == NULL) { goto exit; } ns_total = PyNumber_Add(s_in_ns, ns_fractional); - if (!ns_total) { + if (ns_total == NULL) { goto exit; } PyStructSequence_SET_ITEM(v, ns_index, ns_total); @@ -2649,7 +2649,7 @@ fill_time(PyObject *module, PyObject *v, int s_index, int f_index, int ns_index, } return res; - #undef S_TO_NS + #undef SEC_TO_NS } #ifdef MS_WINDOWS