From 5cfbb3d602b8585370e7c56547ce04d5e4000688 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Filipe=20La=C3=ADns?= Date: Fri, 22 Nov 2024 23:53:35 +0000 Subject: [PATCH 1/6] GH-127178: convert _sysconfigdata to a JSON file MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Filipe Laíns --- Lib/sysconfig/__init__.py | 33 ++++++++++++++++----------------- Lib/sysconfig/__main__.py | 8 +++----- Makefile.pre.in | 2 +- 3 files changed, 20 insertions(+), 23 deletions(-) diff --git a/Lib/sysconfig/__init__.py b/Lib/sysconfig/__init__.py index 67a071963d8c7d..00c6f6643a4e8b 100644 --- a/Lib/sysconfig/__init__.py +++ b/Lib/sysconfig/__init__.py @@ -227,6 +227,10 @@ def is_python_build(check_home=None): return True return False +def _get_pybuilddir(): + with open(os.path.join(_PROJECT_BASE, 'pybuilddir.txt')) as f: + return os.path.join(_PROJECT_BASE, f.read()) + _PYTHON_BUILD = is_python_build() if _PYTHON_BUILD: @@ -337,24 +341,19 @@ def _get_sysconfigdata_name(): def _init_posix(vars): """Initialize the module as appropriate for POSIX systems.""" # _sysconfigdata is generated at build time, see _generate_posix_vars() - name = _get_sysconfigdata_name() - - # For cross builds, the path to the target's sysconfigdata must be specified - # so it can be imported. It cannot be in PYTHONPATH, as foreign modules in - # sys.path can cause crashes when loaded by the host interpreter. - # Rely on truthiness as a valueless env variable is still an empty string. - # See OS X note in _generate_posix_vars re _sysconfigdata. - if (path := os.environ.get('_PYTHON_SYSCONFIGDATA_PATH')): - from importlib.machinery import FileFinder, SourceFileLoader, SOURCE_SUFFIXES - from importlib.util import module_from_spec - spec = FileFinder(path, (SourceFileLoader, SOURCE_SUFFIXES)).find_spec(name) - _temp = module_from_spec(spec) - spec.loader.exec_module(_temp) - else: - _temp = __import__(name, globals(), locals(), ['build_time_vars'], 0) - build_time_vars = _temp.build_time_vars + data_dir = _get_pybuilddir() if is_python_build() else sys._stdlib_dir + data_path = os.environ.get( + '_PYTHON_SYSCONFIGDATA_PATH', + os.path.join(data_dir, _get_sysconfigdata_name() + '.json') + ) + + import json + + with open(data_path) as f: + data = json.load(f) + # GH-126920: Make sure we don't overwrite any of the keys already set - vars.update(build_time_vars | vars) + vars.update(data | vars) def _init_non_posix(vars): """Initialize the module as appropriate for NT""" diff --git a/Lib/sysconfig/__main__.py b/Lib/sysconfig/__main__.py index d7257b9d2d00db..c8464427b68963 100644 --- a/Lib/sysconfig/__main__.py +++ b/Lib/sysconfig/__main__.py @@ -1,3 +1,4 @@ +import json import os import sys from sysconfig import ( @@ -207,13 +208,10 @@ def _generate_posix_vars(): if hasattr(sys, "gettotalrefcount"): pybuilddir += '-pydebug' os.makedirs(pybuilddir, exist_ok=True) - destfile = os.path.join(pybuilddir, name + '.py') + destfile = os.path.join(pybuilddir, name + '.json') with open(destfile, 'w', encoding='utf8') as f: - f.write('# system configuration generated and used by' - ' the sysconfig module\n') - f.write('build_time_vars = ') - _print_config_dict(vars, stream=f) + json.dump(vars, f, indent=2) # Create file used for sys.path fixup -- see Modules/getpath.c with open('pybuilddir.txt', 'w', encoding='utf8') as f: diff --git a/Makefile.pre.in b/Makefile.pre.in index 8d94ba361fd934..4736bcd1bb93ab 100644 --- a/Makefile.pre.in +++ b/Makefile.pre.in @@ -2645,7 +2645,7 @@ libinstall: all $(srcdir)/Modules/xxmodule.c esac; \ done; \ done - $(INSTALL_DATA) `cat pybuilddir.txt`/_sysconfigdata_$(ABIFLAGS)_$(MACHDEP)_$(MULTIARCH).py \ + $(INSTALL_DATA) `cat pybuilddir.txt`/_sysconfigdata_$(ABIFLAGS)_$(MACHDEP)_$(MULTIARCH).json \ $(DESTDIR)$(LIBDEST); \ $(INSTALL_DATA) $(srcdir)/LICENSE $(DESTDIR)$(LIBDEST)/LICENSE.txt @ # If app store compliance has been configured, apply the patch to the From dc0fafabd268e40ce3d56d50f1baa817bc8ecf28 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Filipe=20La=C3=ADns?= Date: Fri, 22 Nov 2024 23:54:56 +0000 Subject: [PATCH 2/6] Fix sysconfig.is_python_build() under virtual environments MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Filipe Laíns --- Lib/sysconfig/__init__.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Lib/sysconfig/__init__.py b/Lib/sysconfig/__init__.py index 00c6f6643a4e8b..ce16b3780e7b5f 100644 --- a/Lib/sysconfig/__init__.py +++ b/Lib/sysconfig/__init__.py @@ -189,8 +189,8 @@ def _safe_realpath(path): except OSError: return path -if sys.executable: - _PROJECT_BASE = os.path.dirname(_safe_realpath(sys.executable)) +if sys._base_executable: + _PROJECT_BASE = os.path.dirname(_safe_realpath(sys._base_executable)) else: # sys.executable can be empty if argv[0] has been changed and Python is # unable to retrieve the real program name From daf670cedb3632e776c1dcca6e8dec8fde72e058 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Filipe=20La=C3=ADns?= Date: Fri, 22 Nov 2024 23:57:29 +0000 Subject: [PATCH 3/6] Add news MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Filipe Laíns --- .../next/Library/2024-11-22-23-57-28.gh-issue-127178.2J-NbL.rst | 2 ++ 1 file changed, 2 insertions(+) create mode 100644 Misc/NEWS.d/next/Library/2024-11-22-23-57-28.gh-issue-127178.2J-NbL.rst diff --git a/Misc/NEWS.d/next/Library/2024-11-22-23-57-28.gh-issue-127178.2J-NbL.rst b/Misc/NEWS.d/next/Library/2024-11-22-23-57-28.gh-issue-127178.2J-NbL.rst new file mode 100644 index 00000000000000..4c122fefa7030c --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-11-22-23-57-28.gh-issue-127178.2J-NbL.rst @@ -0,0 +1,2 @@ +The ``_sysconfigdata_*`` module that holds the :mod:`sysconfig` data on +POSIX has been converted to a JSON file. From 0fa35176ca0d796e15939355f7279c29b3ea399f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Filipe=20La=C3=ADns?= Date: Sat, 23 Nov 2024 00:13:44 +0000 Subject: [PATCH 4/6] Fix sysconfigdata generation on macOS MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Filipe Laíns --- Lib/sysconfig/__init__.py | 31 ++++++++++++++++++++----------- Lib/sysconfig/__main__.py | 11 +++-------- 2 files changed, 23 insertions(+), 19 deletions(-) diff --git a/Lib/sysconfig/__init__.py b/Lib/sysconfig/__init__.py index ce16b3780e7b5f..9789dd4feb4b11 100644 --- a/Lib/sysconfig/__init__.py +++ b/Lib/sysconfig/__init__.py @@ -181,6 +181,7 @@ def joinuser(*args): # True iff _CONFIG_VARS has been fully initialized. _CONFIG_VARS_INITIALIZED = False _USER_BASE = None +_SYSCONFIGDATA = None def _safe_realpath(path): @@ -338,22 +339,30 @@ def _get_sysconfigdata_name(): f'_sysconfigdata_{sys.abiflags}_{sys.platform}_{multiarch}', ) -def _init_posix(vars): - """Initialize the module as appropriate for POSIX systems.""" + +def _get_sysconfigdata(): # _sysconfigdata is generated at build time, see _generate_posix_vars() - data_dir = _get_pybuilddir() if is_python_build() else sys._stdlib_dir - data_path = os.environ.get( - '_PYTHON_SYSCONFIGDATA_PATH', - os.path.join(data_dir, _get_sysconfigdata_name() + '.json') - ) + global _SYSCONFIGDATA + + if _SYSCONFIGDATA is None: + data_dir = _get_pybuilddir() if is_python_build() else sys._stdlib_dir + data_path = os.environ.get( + '_PYTHON_SYSCONFIGDATA_PATH', + os.path.join(data_dir, _get_sysconfigdata_name() + '.json') + ) + + import json + + with open(data_path) as f: + _SYSCONFIGDATA = json.load(f) - import json + return _SYSCONFIGDATA - with open(data_path) as f: - data = json.load(f) +def _init_posix(vars): + """Initialize the module as appropriate for POSIX systems.""" # GH-126920: Make sure we don't overwrite any of the keys already set - vars.update(data | vars) + vars.update(_get_sysconfigdata() | vars) def _init_non_posix(vars): """Initialize the module as appropriate for NT""" diff --git a/Lib/sysconfig/__main__.py b/Lib/sysconfig/__main__.py index c8464427b68963..2d5c71c2f9f26f 100644 --- a/Lib/sysconfig/__main__.py +++ b/Lib/sysconfig/__main__.py @@ -1,6 +1,7 @@ import json import os import sys +import sysconfig from sysconfig import ( _ALWAYS_STR, _PYTHON_BUILD, @@ -193,16 +194,10 @@ def _generate_posix_vars(): # _sysconfigdata.py module being constructed. Unfortunately, # get_config_vars() eventually calls _init_posix(), which attempts # to import _sysconfigdata, which we won't have built yet. In order - # for _init_posix() to work, if we're on Darwin, just mock up the - # _sysconfigdata module manually and populate it with the build vars. - # This is more than sufficient for ensuring the subsequent call to - # get_platform() succeeds. + # for _init_posix() to work, we set the _SYSCONFIGDATA cache directly name = _get_sysconfigdata_name() if 'darwin' in sys.platform: - import types - module = types.ModuleType(name) - module.build_time_vars = vars - sys.modules[name] = module + sysconfig._SYSCONFIGDATA = vars pybuilddir = f'build/lib.{get_platform()}-{get_python_version()}' if hasattr(sys, "gettotalrefcount"): From d8ffbb4b28fe727e7b61c8e563774e1f97ed33bf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Filipe=20La=C3=ADns?= Date: Sat, 23 Nov 2024 00:30:05 +0000 Subject: [PATCH 5/6] Search sys.path for _sysconfigdata instead MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Filipe Laíns --- Lib/sysconfig/__init__.py | 27 ++++++++++++++++----------- 1 file changed, 16 insertions(+), 11 deletions(-) diff --git a/Lib/sysconfig/__init__.py b/Lib/sysconfig/__init__.py index 9789dd4feb4b11..aed1482639d655 100644 --- a/Lib/sysconfig/__init__.py +++ b/Lib/sysconfig/__init__.py @@ -190,8 +190,8 @@ def _safe_realpath(path): except OSError: return path -if sys._base_executable: - _PROJECT_BASE = os.path.dirname(_safe_realpath(sys._base_executable)) +if sys.executable: + _PROJECT_BASE = os.path.dirname(_safe_realpath(sys.executable)) else: # sys.executable can be empty if argv[0] has been changed and Python is # unable to retrieve the real program name @@ -228,10 +228,6 @@ def is_python_build(check_home=None): return True return False -def _get_pybuilddir(): - with open(os.path.join(_PROJECT_BASE, 'pybuilddir.txt')) as f: - return os.path.join(_PROJECT_BASE, f.read()) - _PYTHON_BUILD = is_python_build() if _PYTHON_BUILD: @@ -345,11 +341,20 @@ def _get_sysconfigdata(): global _SYSCONFIGDATA if _SYSCONFIGDATA is None: - data_dir = _get_pybuilddir() if is_python_build() else sys._stdlib_dir - data_path = os.environ.get( - '_PYTHON_SYSCONFIGDATA_PATH', - os.path.join(data_dir, _get_sysconfigdata_name() + '.json') - ) + if '_PYTHON_SYSCONFIGDATA_PATH' in os.environ: + data_path = os.environ['_PYTHON_SYSCONFIGDATA_PATH'] + else: + # Search sys.path + # FIXME: We should not need this if we could reliably identify the project directory on source builds + name = _get_sysconfigdata_name() + '.json' + for path in sys.path: + data_path = os.path.join(path, name) + if os.path.isfile(data_path): + break + else: + import warnings + warnings.warn(f'Could not find {name}', RuntimeWarning) + return {} import json From a8ceb3ba5e4e230523bcd088181947d009701feb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Filipe=20La=C3=ADns?= Date: Sat, 23 Nov 2024 00:36:25 +0000 Subject: [PATCH 6/6] Fix _PYTHON_SYSCONFIGDATA_PATH MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Filipe Laíns --- Lib/sysconfig/__init__.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Lib/sysconfig/__init__.py b/Lib/sysconfig/__init__.py index aed1482639d655..0ccfa3da38e6b4 100644 --- a/Lib/sysconfig/__init__.py +++ b/Lib/sysconfig/__init__.py @@ -341,12 +341,12 @@ def _get_sysconfigdata(): global _SYSCONFIGDATA if _SYSCONFIGDATA is None: + name = _get_sysconfigdata_name() + '.json' if '_PYTHON_SYSCONFIGDATA_PATH' in os.environ: - data_path = os.environ['_PYTHON_SYSCONFIGDATA_PATH'] + data_path = os.path.join(os.environ['_PYTHON_SYSCONFIGDATA_PATH'], name) else: # Search sys.path # FIXME: We should not need this if we could reliably identify the project directory on source builds - name = _get_sysconfigdata_name() + '.json' for path in sys.path: data_path = os.path.join(path, name) if os.path.isfile(data_path):