From b228f624ce20a342bdb904164b2d2c38b1064f0f Mon Sep 17 00:00:00 2001 From: "Gregory P. Smith [Google LLC]" Date: Thu, 21 Oct 2021 17:14:47 +0000 Subject: [PATCH 1/6] Raise RLIMIT_NOFILE in test.regrtest. On macOS the default is often too low for our testsuite to succeed. --- Lib/test/libregrtest/main.py | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/Lib/test/libregrtest/main.py b/Lib/test/libregrtest/main.py index 52cc065da115d9..a56a4472d3d3c6 100644 --- a/Lib/test/libregrtest/main.py +++ b/Lib/test/libregrtest/main.py @@ -633,6 +633,24 @@ def cleanup(self): print("Remove file: %s" % name) os_helper.unlink(name) + def adjust_resource_limits(self): + try: + import resource + from resource import RLIMIT_NOFILE, RLIM_INFINITY + except ImportError: + return + fd_limit, max_fds = resource.getrlimit(RLIMIT_NOFILE) + if fd_limit < 2000 and fd_limit < max_fds: + # On macOS the default fd limit is sometimes too low (256) for our + # test suite to succeed. Raise it to something more reasonable. + new_fd_limit = min(2000, max_fds) + try: + resource.setrlimit(RLIMIT_NOFILE, (new_fd_limit, max_fds)) + print(f"Raised RLIMIT_NOFILE to {new_fd_limit}.") + except (ValueError, OSError) as err: + print(f"Unable to raise RLIMIT_NOFILE from {fd_limit} to " + f"{new_fd_limit}: {err}".) + def main(self, tests=None, **kwargs): self.parse_args(kwargs) From 0fb80302e82f4f2603f0467b045144c141ddd85e Mon Sep 17 00:00:00 2001 From: "Gregory P. Smith" Date: Thu, 21 Oct 2021 14:13:55 -0700 Subject: [PATCH 2/6] syntax typo fix --- Lib/test/libregrtest/main.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Lib/test/libregrtest/main.py b/Lib/test/libregrtest/main.py index a56a4472d3d3c6..d82cb4222fcfaa 100644 --- a/Lib/test/libregrtest/main.py +++ b/Lib/test/libregrtest/main.py @@ -649,7 +649,7 @@ def adjust_resource_limits(self): print(f"Raised RLIMIT_NOFILE to {new_fd_limit}.") except (ValueError, OSError) as err: print(f"Unable to raise RLIMIT_NOFILE from {fd_limit} to " - f"{new_fd_limit}: {err}".) + f"{new_fd_limit}: {err}.") def main(self, tests=None, **kwargs): self.parse_args(kwargs) From dffec915dd9150746638c56f8012ef2264cec85f Mon Sep 17 00:00:00 2001 From: "Gregory P. Smith" Date: Thu, 21 Oct 2021 16:06:45 -0700 Subject: [PATCH 3/6] Actually call the new method... --- Lib/test/libregrtest/main.py | 1 + 1 file changed, 1 insertion(+) diff --git a/Lib/test/libregrtest/main.py b/Lib/test/libregrtest/main.py index d82cb4222fcfaa..fc4fdb017a004d 100644 --- a/Lib/test/libregrtest/main.py +++ b/Lib/test/libregrtest/main.py @@ -661,6 +661,7 @@ def main(self, tests=None, **kwargs): sys.exit(0) test_cwd = self.create_temp_dir() + self.adjust_resource_limits() try: # Run the tests in a context manager that temporarily changes the CWD From fc45c29024ef9e8a34ae9362b04660c224bdebae Mon Sep 17 00:00:00 2001 From: "Gregory P. Smith" Date: Thu, 21 Oct 2021 16:59:44 -0700 Subject: [PATCH 4/6] Minor refactoring, use 1024 to match a common Linux distro default. --- Lib/test/libregrtest/main.py | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/Lib/test/libregrtest/main.py b/Lib/test/libregrtest/main.py index fc4fdb017a004d..81cfade23adc6b 100644 --- a/Lib/test/libregrtest/main.py +++ b/Lib/test/libregrtest/main.py @@ -640,13 +640,15 @@ def adjust_resource_limits(self): except ImportError: return fd_limit, max_fds = resource.getrlimit(RLIMIT_NOFILE) - if fd_limit < 2000 and fd_limit < max_fds: - # On macOS the default fd limit is sometimes too low (256) for our - # test suite to succeed. Raise it to something more reasonable. - new_fd_limit = min(2000, max_fds) + # On macOS the default fd limit is sometimes too low (256) for our + # test suite to succeed. Raise it to something more reasonable. + # 1024 is a common Linux default. + desired_fds = 1024 + if fd_limit < desired_fds and fd_limit < max_fds: + new_fd_limit = min(desired_fds, max_fds) try: resource.setrlimit(RLIMIT_NOFILE, (new_fd_limit, max_fds)) - print(f"Raised RLIMIT_NOFILE to {new_fd_limit}.") + print(f"Raised RLIMIT_NOFILE: {fd_limit} -> {new_fd_limit}") except (ValueError, OSError) as err: print(f"Unable to raise RLIMIT_NOFILE from {fd_limit} to " f"{new_fd_limit}: {err}.") From 508e12b0930016df712f46ff0fd1db7cd2ceb4c3 Mon Sep 17 00:00:00 2001 From: "Gregory P. Smith" Date: Thu, 21 Oct 2021 17:16:20 -0700 Subject: [PATCH 5/6] Move the code to libregrtest.setup setup_tests(). --- Lib/test/libregrtest/main.py | 21 --------------------- Lib/test/libregrtest/setup.py | 24 ++++++++++++++++++++++++ 2 files changed, 24 insertions(+), 21 deletions(-) diff --git a/Lib/test/libregrtest/main.py b/Lib/test/libregrtest/main.py index 81cfade23adc6b..52cc065da115d9 100644 --- a/Lib/test/libregrtest/main.py +++ b/Lib/test/libregrtest/main.py @@ -633,26 +633,6 @@ def cleanup(self): print("Remove file: %s" % name) os_helper.unlink(name) - def adjust_resource_limits(self): - try: - import resource - from resource import RLIMIT_NOFILE, RLIM_INFINITY - except ImportError: - return - fd_limit, max_fds = resource.getrlimit(RLIMIT_NOFILE) - # On macOS the default fd limit is sometimes too low (256) for our - # test suite to succeed. Raise it to something more reasonable. - # 1024 is a common Linux default. - desired_fds = 1024 - if fd_limit < desired_fds and fd_limit < max_fds: - new_fd_limit = min(desired_fds, max_fds) - try: - resource.setrlimit(RLIMIT_NOFILE, (new_fd_limit, max_fds)) - print(f"Raised RLIMIT_NOFILE: {fd_limit} -> {new_fd_limit}") - except (ValueError, OSError) as err: - print(f"Unable to raise RLIMIT_NOFILE from {fd_limit} to " - f"{new_fd_limit}: {err}.") - def main(self, tests=None, **kwargs): self.parse_args(kwargs) @@ -663,7 +643,6 @@ def main(self, tests=None, **kwargs): sys.exit(0) test_cwd = self.create_temp_dir() - self.adjust_resource_limits() try: # Run the tests in a context manager that temporarily changes the CWD diff --git a/Lib/test/libregrtest/setup.py b/Lib/test/libregrtest/setup.py index f9460ae7ed18f2..4ffd15478f6437 100644 --- a/Lib/test/libregrtest/setup.py +++ b/Lib/test/libregrtest/setup.py @@ -39,6 +39,7 @@ def setup_tests(ns): for signum in signals: faulthandler.register(signum, chain=True, file=stderr_fd) + _adjust_resource_limits() replace_stdout() support.record_original_stdout(sys.stdout) @@ -133,3 +134,26 @@ def restore_stdout(): sys.stdout.close() sys.stdout = stdout atexit.register(restore_stdout) + + +def _adjust_resource_limits(): + """Adjust the system resource limits (ulimit) if needed.""" + try: + import resource + from resource import RLIMIT_NOFILE, RLIM_INFINITY + except ImportError: + return + fd_limit, max_fds = resource.getrlimit(RLIMIT_NOFILE) + # On macOS the default fd limit is sometimes too low (256) for our + # test suite to succeed. Raise it to something more reasonable. + # 1024 is a common Linux default. + desired_fds = 1024 + if fd_limit < desired_fds and fd_limit < max_fds: + new_fd_limit = min(desired_fds, max_fds) + try: + resource.setrlimit(RLIMIT_NOFILE, (new_fd_limit, max_fds)) + print(f"Raised RLIMIT_NOFILE: {fd_limit} -> {new_fd_limit}") + except (ValueError, OSError) as err: + print(f"Unable to raise RLIMIT_NOFILE from {fd_limit} to " + f"{new_fd_limit}: {err}.") + From e434a4a13ffa0aea2324c8d79a7c3a7550b19c50 Mon Sep 17 00:00:00 2001 From: "Gregory P. Smith" Date: Thu, 21 Oct 2021 17:22:47 -0700 Subject: [PATCH 6/6] Add a NEWS entry. --- .../NEWS.d/next/Tests/2021-10-21-17-22-26.bpo-43592.kHRsra.rst | 3 +++ 1 file changed, 3 insertions(+) create mode 100644 Misc/NEWS.d/next/Tests/2021-10-21-17-22-26.bpo-43592.kHRsra.rst diff --git a/Misc/NEWS.d/next/Tests/2021-10-21-17-22-26.bpo-43592.kHRsra.rst b/Misc/NEWS.d/next/Tests/2021-10-21-17-22-26.bpo-43592.kHRsra.rst new file mode 100644 index 00000000000000..2528857caf9c04 --- /dev/null +++ b/Misc/NEWS.d/next/Tests/2021-10-21-17-22-26.bpo-43592.kHRsra.rst @@ -0,0 +1,3 @@ +:mod:`test.libregrtest` now raises the soft resource limit for the maximum +number of file descriptors when the default is too low for our test suite as +was often the case on macOS.