From ce35247c8e93cc1bf9c05b78d7553658ccb00615 Mon Sep 17 00:00:00 2001 From: Peter Bierma Date: Sun, 18 May 2025 22:19:10 -0400 Subject: [PATCH 01/13] Add support for Python 3.14 Signed-off-by: Peter Bierma --- .github/workflows/build_wheels.yml | 6 +- news/229.feature.rst | 1 + src/pystack/_pystack/cpython/code.h | 39 ++++ src/pystack/_pystack/cpython/frame.h | 26 +++ src/pystack/_pystack/cpython/gc.h | 24 +++ src/pystack/_pystack/cpython/interpreter.h | 78 ++++++++ src/pystack/_pystack/cpython/runtime.h | 209 +++++++++++++++++++++ src/pystack/_pystack/cpython/thread.h | 67 +++++++ src/pystack/_pystack/process.cpp | 8 + src/pystack/_pystack/process.h | 2 + src/pystack/_pystack/pycode.cpp | 34 +++- src/pystack/_pystack/pycode.h | 3 +- src/pystack/_pystack/pyframe.cpp | 11 +- src/pystack/_pystack/version.cpp | 47 ++++- 14 files changed, 544 insertions(+), 11 deletions(-) create mode 100644 news/229.feature.rst diff --git a/.github/workflows/build_wheels.yml b/.github/workflows/build_wheels.yml index 08be5b32..e3aa9f74 100644 --- a/.github/workflows/build_wheels.yml +++ b/.github/workflows/build_wheels.yml @@ -68,7 +68,7 @@ jobs: - name: Build wheels uses: pypa/cibuildwheel@v3.0.1 env: - CIBW_BUILD: "cp3{8..13}-${{ matrix.wheel_type }}" + CIBW_BUILD: "cp3{8..14}-${{ matrix.wheel_type }}" CIBW_ARCHS_LINUX: auto aarch64 CIBW_ENABLE: cpython-prerelease - uses: actions/upload-artifact@v4 @@ -122,7 +122,7 @@ jobs: strategy: fail-fast: false matrix: - python_version: ["3.8", "3.9", "3.10", "3.11", "3.12", "3.13"] + python_version: ["3.8", "3.9", "3.10", "3.11", "3.12", "3.13", "3.14"] steps: - uses: actions/checkout@v4 - name: Set up Python @@ -160,7 +160,7 @@ jobs: strategy: fail-fast: false matrix: - python_version: ["3.13"] + python_version: ["3.13", "3.14"] steps: - uses: actions/checkout@v4 - name: Set up Python diff --git a/news/229.feature.rst b/news/229.feature.rst new file mode 100644 index 00000000..35b51feb --- /dev/null +++ b/news/229.feature.rst @@ -0,0 +1 @@ +Add support for Python 3.14 diff --git a/src/pystack/_pystack/cpython/code.h b/src/pystack/_pystack/cpython/code.h index 5069ffe7..d14db7de 100644 --- a/src/pystack/_pystack/cpython/code.h +++ b/src/pystack/_pystack/cpython/code.h @@ -197,4 +197,43 @@ typedef struct } PyCodeObject; } // namespace Python3_13 +namespace Python3_14 { +typedef uint16_t _Py_CODEUNIT; + +typedef struct +{ + PyObject_VAR_HEAD PyObject* co_consts; + PyObject* co_names; + PyObject* co_exceptiontable; + int co_flags; + int co_argcount; + int co_posonlyargcount; + int co_kwonlyargcount; + int co_stacksize; + int co_firstlineno; + int co_nlocalsplus; + int co_framesize; + int co_nlocals; + int co_ncellvars; + int co_nfreevars; + uint32_t co_version; + PyObject* co_localsplusnames; + PyObject* co_localspluskinds; + PyObject* co_filename; + PyObject* co_name; + PyObject* co_qualname; + PyObject* co_linetable; + PyObject* co_weakreflist; + void* co_executors; + void* _co_cached; + uintptr_t _co_instrumentation_version; + void* _co_monitoring; + Py_ssize_t _co_unique_id; + int _co_firsttraceable; + void* co_extra; + /* deal with co_tlbc somehow */ + char co_code_adaptive[1]; +} PyCodeObject; +} // namespace Python3_14 + } // namespace pystack diff --git a/src/pystack/_pystack/cpython/frame.h b/src/pystack/_pystack/cpython/frame.h index b31c58b1..c2299907 100644 --- a/src/pystack/_pystack/cpython/frame.h +++ b/src/pystack/_pystack/cpython/frame.h @@ -126,4 +126,30 @@ typedef struct _interpreter_frame } // namespace Python3_12 +namespace Python3_14 { + +typedef union _PyStackRef { + uintptr_t bits; +} _PyStackRef; + +typedef struct _interpreter_frame +{ + _PyStackRef f_executable; + void* previous; + void* f_funcobj; + PyObject* f_globals; + PyObject* f_builtins; + PyObject* f_locals; + PyObject* frame_obj; + _Py_CODEUNIT* instr_ptr; + _PyStackRef stackpointer; + /* int32_t tlbc_index; */ + uint16_t return_offset; + char owner; + uint8_t visited; + void* localsplus[1]; +} PyFrameObject; + +} // namespace Python3_14 + } // namespace pystack diff --git a/src/pystack/_pystack/cpython/gc.h b/src/pystack/_pystack/cpython/gc.h index 3b536ceb..b64bd073 100644 --- a/src/pystack/_pystack/cpython/gc.h +++ b/src/pystack/_pystack/cpython/gc.h @@ -82,4 +82,28 @@ struct _gc_runtime_state }; } // namespace Python3_8 + +namespace Python3_14 { + +struct _gc_runtime_state +{ + PyObject* trash_delete_later; + int trash_delete_nesting; + int enabled; + int debug; + struct Python3_8::gc_generation young; + struct Python3_8::gc_generation old[2]; + struct Python3_8::gc_generation permanent_generation; + struct gc_generation_stats generation_stats[NUM_GENERATIONS]; + int collecting; + PyObject* garbage; + PyObject* callbacks; + Py_ssize_t heap_size; + Py_ssize_t work_to_do; + int visited_space; + int phase; +}; + +} // namespace Python3_14 + } // namespace pystack diff --git a/src/pystack/_pystack/cpython/interpreter.h b/src/pystack/_pystack/cpython/interpreter.h index 719116c3..b35b26ba 100644 --- a/src/pystack/_pystack/cpython/interpreter.h +++ b/src/pystack/_pystack/cpython/interpreter.h @@ -338,4 +338,82 @@ typedef struct _is struct _import_state imports; } PyInterpreterState; } // namespace Python3_13 + +namespace Python3_14 { + +struct _pythreadstate; + +typedef struct +{ + Python3_13::PyMutex mutex; + unsigned long long thread; + size_t level; +} _PyRecursiveMutex; + +struct _import_state +{ + PyObject* modules; + PyObject* modules_by_index; + PyObject* importlib; + int override_frozen_modules; + int override_multi_interp_extensions_check; + PyObject* import_func; + _PyRecursiveMutex lock; + /* diagnostic info in PyImport_ImportModuleLevelObject() */ + struct + { + int import_level; + int64_t accumulated; + int header; + } find_and_load; +}; + +struct _gil_runtime_state +{ + unsigned long interval; + struct _pythreadstate* last_holder; + int locked; + unsigned long switch_number; + pthread_cond_t cond; + pthread_cond_t mutex; +#ifdef FORCE_SWITCHING + pthread_cond_t switch_cond; + pthread_cond_t switch_mutex; +#endif +}; + +typedef struct _is +{ + struct _ceval_state ceval; + void* _malloced; + struct _is* next; + int64_t id; + Py_ssize_t id_refcount; + int requires_idref; + long _whence; + int _initialized; + int _ready; + int finalizing; + uintptr_t last_restart_version; + struct pythreads + { + uint64_t next_unique_id; + struct _pythreadstate* head; + struct _pythreadstate* preallocated; + struct _pythreadstate* main; + Py_ssize_t count; + size_t stacksize; + } threads; + void* runtime; + struct _pythreadstate* _finalizing; + unsigned long _finalizing_id; + struct _gc_runtime_state gc; + PyObject* sysdict; + PyObject* builtins; + struct _import_state imports; + struct _gil_runtime_state _gil; +} PyInterpreterState; + +} // namespace Python3_14 + } // namespace pystack diff --git a/src/pystack/_pystack/cpython/runtime.h b/src/pystack/_pystack/cpython/runtime.h index b03d9c53..2d9adf86 100644 --- a/src/pystack/_pystack/cpython/runtime.h +++ b/src/pystack/_pystack/cpython/runtime.h @@ -422,6 +422,214 @@ typedef struct pyruntimestate } PyRuntimeState; } // namespace Python3_13 + +namespace Python3_14 { + +typedef struct _Py_DebugOffsets +{ + char cookie[8]; + uint64_t version; + uint64_t free_threaded; + // Runtime state offset; + struct _runtime_state + { + uint64_t size; + uint64_t finalizing; + uint64_t interpreters_head; + } runtime_state; + + // Interpreter state offset; + struct _interpreter_state + { + uint64_t size; + uint64_t id; + uint64_t next; + uint64_t threads_head; + uint64_t threads_main; + uint64_t gc; + uint64_t imports_modules; + uint64_t sysdict; + uint64_t builtins; + uint64_t ceval_gil; + uint64_t gil_runtime_state; + uint64_t gil_runtime_state_enabled; + uint64_t gil_runtime_state_locked; + uint64_t gil_runtime_state_holder; + } interpreter_state; + + // Thread state offset; + struct _thread_state + { + uint64_t size; + uint64_t prev; + uint64_t next; + uint64_t interp; + uint64_t current_frame; + uint64_t thread_id; + uint64_t native_thread_id; + uint64_t datastack_chunk; + uint64_t status; + } thread_state; + + // InterpreterFrame offset; + struct _interpreter_frame + { + uint64_t size; + uint64_t previous; + uint64_t executable; + uint64_t instr_ptr; + uint64_t localsplus; + uint64_t owner; + uint64_t stackpointer; + uint64_t tlbc_index; + } interpreter_frame; + + // Code object offset; + struct _code_object + { + uint64_t size; + uint64_t filename; + uint64_t name; + uint64_t qualname; + uint64_t linetable; + uint64_t firstlineno; + uint64_t argcount; + uint64_t localsplusnames; + uint64_t localspluskinds; + uint64_t co_code_adaptive; + uint64_t co_tlbc; + } code_object; + + // PyObject offset; + struct _pyobject + { + uint64_t size; + uint64_t ob_type; + } pyobject; + + // PyTypeObject object offset; + struct _type_object + { + uint64_t size; + uint64_t tp_name; + uint64_t tp_repr; + uint64_t tp_flags; + } type_object; + + // PyTuple object offset; + struct _tuple_object + { + uint64_t size; + uint64_t ob_item; + uint64_t ob_size; + } tuple_object; + + // PyList object offset; + struct _list_object + { + uint64_t size; + uint64_t ob_item; + uint64_t ob_size; + } list_object; + + // PySet object offset; + struct _set_object + { + uint64_t size; + uint64_t used; + uint64_t table; + uint64_t mask; + } set_object; + + // PyDict object offset; + struct _dict_object + { + uint64_t size; + uint64_t ma_keys; + uint64_t ma_values; + } dict_object; + + // PyFloat object offset; + struct _float_object + { + uint64_t size; + uint64_t ob_fval; + } float_object; + + // PyLong object offset; + struct _long_object + { + uint64_t size; + uint64_t lv_tag; + uint64_t ob_digit; + } long_object; + + // PyBytes object offset; + struct _bytes_object + { + uint64_t size; + uint64_t ob_size; + uint64_t ob_sval; + } bytes_object; + + // Unicode object offset; + struct _unicode_object + { + uint64_t size; + uint64_t state; + uint64_t length; + uint64_t asciiobject_size; + } unicode_object; + + // GC runtime state offset; + struct _gc + { + uint64_t size; + uint64_t collecting; + } gc; + + // Generator object offset; + struct _gen_object + { + uint64_t size; + uint64_t gi_name; + uint64_t gi_iframe; + uint64_t gi_frame_state; + } gen_object; + + struct _debugger_support + { + uint64_t eval_breaker; + uint64_t remote_debugger_support; + uint64_t remote_debugging_enabled; + uint64_t debugger_pending_call; + uint64_t debugger_script_path; + uint64_t debugger_script_path_size; + } debugger_support; +} _Py_DebugOffsets; + +typedef struct pyruntimestate +{ + _Py_DebugOffsets debug_offsets; + int _initialized; + int preinitializing; + int preinitialized; + int core_initialized; + int initialized; + struct _pythreadstate* finalizing; + unsigned long _finalizing_id; + + struct pyinterpreters + { + Python3_13::PyMutex mutex; + PyInterpreterState* head; + PyInterpreterState* main; + int64_t next_id; + } interpreters; +} PyRuntimeState; + +} // namespace Python3_14 + typedef union { Python3_7::PyRuntimeState v3_7; Python3_8::PyRuntimeState v3_8; @@ -429,6 +637,7 @@ typedef union { Python3_11::PyRuntimeState v3_11; Python3_12::PyRuntimeState v3_12; Python3_13::PyRuntimeState v3_13; + Python3_14::PyRuntimeState v3_14; } PyRuntimeState; } // namespace pystack diff --git a/src/pystack/_pystack/cpython/thread.h b/src/pystack/_pystack/cpython/thread.h index c9b5da8d..977cd2ae 100644 --- a/src/pystack/_pystack/cpython/thread.h +++ b/src/pystack/_pystack/cpython/thread.h @@ -295,4 +295,71 @@ typedef struct _pythreadstate } PyThreadState; } // namespace Python3_13 +namespace Python3_14 { + +typedef struct _remote_debugger_support +{ + int32_t debugger_pending_call; + char debugger_script_path[512]; +} _PyRemoteDebuggerSupport; + +typedef struct _pythreadstate +{ + struct _pythreadstate* prev; + struct _pythreadstate* next; + PyInterpreterState* interp; + uintptr_t eval_breaker; + struct + { + unsigned int initialized : 1; + unsigned int bound : 1; + unsigned int unbound : 1; + unsigned int bound_gilstate : 1; + unsigned int active : 1; + unsigned int finalizing : 1; + unsigned int cleared : 1; + unsigned int finalized : 1; + unsigned int : 24; + } _status; + int holds_gil; + int _whence; + int state; + int py_recursion_remaining; + int py_recursion_limit; + int recursion_headroom; + int tracing; + int what_event; + struct _PyInterpreterFrame* current_frame; + Py_tracefunc c_profilefunc; + Py_tracefunc c_tracefunc; + PyObject* c_profileobj; + PyObject* c_traceobj; + PyObject* current_exception; + Python3_13::_PyErr_StackItem* exc_info; + PyObject* dict; + int gilstate_counter; + PyObject* async_exc; + unsigned long thread_id; + unsigned long native_thread_id; + PyObject* delete_later; + uintptr_t critical_section; + int coroutine_origin_tracking_depth; + PyObject* async_gen_firstiter; + PyObject* async_gen_finalizer; + PyObject* context; + uint64_t context_ver; + uint64_t id; + void* datastack_chunk; + PyObject** datastack_top; + PyObject** datastack_limit; + Python3_13::_PyErr_StackItem exc_state; + PyObject* current_executor; + uint64_t dict_global_version; + PyObject* threading_local_key; + PyObject* threading_local_sentinel; + _PyRemoteDebuggerSupport remote_debugger_support; +} PyThreadState; + +} // namespace Python3_14 + } // namespace pystack diff --git a/src/pystack/_pystack/process.cpp b/src/pystack/_pystack/process.cpp index b937cef5..b48a2252 100644 --- a/src/pystack/_pystack/process.cpp +++ b/src/pystack/_pystack/process.cpp @@ -706,12 +706,14 @@ AbstractProcessManager::setPythonVersionFromDebugOffsets() << " identify the version as " << parsed; setPythonVersion(std::make_pair(parsed.major, parsed.minor)); Structure py_runtime(shared_from_this(), pyruntime_addr); + bool is_free_threaded = py_runtime.getField(&py_runtime_v::o_dbg_off_free_threaded); std::unique_ptr offsets = loadDebugOffsets(py_runtime); if (offsets) { LOG(INFO) << "_Py_DebugOffsets appear to be valid and will be used"; warnIfOffsetsAreMismatched(pyruntime_addr); d_debug_offsets_addr = pyruntime_addr; d_debug_offsets = std::move(offsets); + d_is_free_threaded = is_free_threaded; return; } } @@ -1279,6 +1281,12 @@ AbstractProcessManager::versionIsAtLeast(int required_major, int required_minor) return d_major > required_major || (d_major == required_major && d_minor >= required_minor); } +bool +AbstractProcessManager::isFreeThreaded() const +{ + return d_is_free_threaded; +} + const python_v& AbstractProcessManager::offsets() const { diff --git a/src/pystack/_pystack/process.h b/src/pystack/_pystack/process.h index acf3dbe9..e8554b02 100644 --- a/src/pystack/_pystack/process.h +++ b/src/pystack/_pystack/process.h @@ -95,6 +95,7 @@ class AbstractProcessManager : public std::enable_shared_from_this& version); bool versionIsAtLeast(int required_major, int required_minor) const; + bool isFreeThreaded() const; const python_v& offsets() const; protected: @@ -111,6 +112,7 @@ class AbstractProcessManager : public std::enable_shared_from_this d_debug_offsets{}; mutable std::unordered_map d_type_cache; diff --git a/src/pystack/_pystack/pycode.cpp b/src/pystack/_pystack/pycode.cpp index 8d8c9505..fc490e37 100644 --- a/src/pystack/_pystack/pycode.cpp +++ b/src/pystack/_pystack/pycode.cpp @@ -106,7 +106,8 @@ getLocationInfo( const std::shared_ptr& manager, remote_addr_t code_addr, Structure& code, - uintptr_t last_instruction_index) + uintptr_t last_instruction_index, + int tlbc_index) { int code_lineno = code.getField(&py_code_v::o_firstlineno); remote_addr_t lnotab_addr = code.getField(&py_code_v::o_lnotab); @@ -120,7 +121,31 @@ getLocationInfo( // Check out https://github.com/python/cpython/blob/main/Objects/lnotab_notes.txt for the format of // the lnotab table in different versions of the interpreter. - if (manager->versionIsAtLeast(3, 11)) { + if (manager->versionIsAtLeast(3, 14) && manager->isFreeThreaded()) { + uintptr_t code_adaptive = code.getFieldRemoteAddress(&py_code_v::o_code_adaptive); + uintptr_t tlbc_entries_addr = code_adaptive - sizeof(void*); + uintptr_t tlbc_entries; + manager->copyMemoryFromProcess(tlbc_entries_addr, sizeof(tlbc_entries), &tlbc_entries); + Py_ssize_t tlbc_size; + manager->copyMemoryFromProcess(tlbc_entries, sizeof(tlbc_size), &tlbc_size); + std::vector vec(tlbc_size); + manager->copyMemoryFromProcess( + tlbc_entries + sizeof(tlbc_size), + tlbc_size * sizeof(uintptr_t), + vec.data()); + uintptr_t code_adaptive_actual = vec[tlbc_index]; + ptrdiff_t addrq = + (reinterpret_cast(last_instruction_index) + - reinterpret_cast(code_adaptive_actual)); + LocationInfo posinfo; + bool ret = parse_linetable(addrq, lnotab, code_lineno, &posinfo); + if (ret) { + location_info.lineno = posinfo.lineno; + location_info.end_lineno = posinfo.end_lineno; + location_info.column = posinfo.column; + location_info.end_column = posinfo.end_column; + } + } else if (manager->versionIsAtLeast(3, 11)) { uintptr_t code_adaptive = code.getFieldRemoteAddress(&py_code_v::o_code_adaptive); ptrdiff_t addrq = (reinterpret_cast(last_instruction_index) @@ -165,7 +190,8 @@ getLocationInfo( CodeObject::CodeObject( const std::shared_ptr& manager, remote_addr_t addr, - uintptr_t lasti) + uintptr_t lasti, + int tlbc_index) { LOG(DEBUG) << std::hex << std::showbase << "Copying code struct from address " << addr; Structure code(manager, addr); @@ -183,7 +209,7 @@ CodeObject::CodeObject( LOG(DEBUG) << "Code object scope: " << d_filename; LOG(DEBUG) << "Obtaining location info location"; - d_location_info = getLocationInfo(manager, addr, code, lasti); + d_location_info = getLocationInfo(manager, addr, code, lasti, tlbc_index); LOG(DEBUG) << "Code object location info: line_range=(" << d_location_info.lineno << ", " << d_location_info.end_lineno << ") column_range=(" << d_location_info.column << ", " << d_location_info.end_column << ")"; diff --git a/src/pystack/_pystack/pycode.h b/src/pystack/_pystack/pycode.h index c4a2706b..e58fa00c 100644 --- a/src/pystack/_pystack/pycode.h +++ b/src/pystack/_pystack/pycode.h @@ -26,7 +26,8 @@ class CodeObject CodeObject( const std::shared_ptr& manager, remote_addr_t addr, - uintptr_t lastli); + uintptr_t lastli, + int tlbc_index); CodeObject(std::string filename, std::string scope, LocationInfo location_info); // Getters diff --git a/src/pystack/_pystack/pyframe.cpp b/src/pystack/_pystack/pyframe.cpp index b8225ec7..e26e7d7b 100644 --- a/src/pystack/_pystack/pyframe.cpp +++ b/src/pystack/_pystack/pyframe.cpp @@ -59,6 +59,9 @@ FrameObject::getCode( Structure& frame) { remote_addr_t py_code_addr = frame.getField(&py_frame_v::o_code); + if (manager->versionIsAtLeast(3, 14)) { + py_code_addr = py_code_addr & (~3); + } LOG(DEBUG) << std::hex << std::showbase << "Attempting to construct code object from address " << py_code_addr; @@ -69,8 +72,14 @@ FrameObject::getCode( } else { last_instruction = frame.getField(&py_frame_v::o_lasti); } + int32_t tlbc_index = -1; + if (manager->versionIsAtLeast(3, 14) && manager->isFreeThreaded()) { + uintptr_t tlbc_index_addr = frame.getFieldRemoteAddress(&py_frame_v::o_prev_instr) + + sizeof(last_instruction) + sizeof(Python3_14::_PyStackRef); + manager->copyMemoryFromProcess(tlbc_index_addr, sizeof(tlbc_index), &tlbc_index); + } try { - return std::make_unique(manager, py_code_addr, last_instruction); + return std::make_unique(manager, py_code_addr, last_instruction, tlbc_index); } catch (const RemoteMemCopyError& ex) { // This may not have been a code object at all, or it may have been // trashed by memory corruption. Either way, indicate that we failed diff --git a/src/pystack/_pystack/version.cpp b/src/pystack/_pystack/version.cpp index 2f76c77d..f58ff878 100644 --- a/src/pystack/_pystack/version.cpp +++ b/src/pystack/_pystack/version.cpp @@ -89,6 +89,22 @@ py_framev312() }; } +template +constexpr py_frame_v +py_framev314() +{ + return { + sizeof(T), + {offsetof(T, previous)}, + {offsetof(T, f_executable)}, + {0}, + {offsetof(T, instr_ptr)}, + {offsetof(T, localsplus)}, + {0}, + {offsetof(T, owner)}, + }; +} + template constexpr py_thread_v py_thead_h() @@ -704,6 +720,30 @@ python_v python_v3_13 = { py_gilruntimestate(), }; +// ---- Python 3.14 ------------------------------------------------------------ + +python_v python_v3_14 = { + py_tuple(), + py_list(), + py_dict(), + py_dictkeys(), + py_dictvalues(), + py_float(), + py_long<_PyLongObject>(), + py_bytes(), + py_unicode(), + py_object(), + py_type(), + py_codev311(), + py_framev314(), + py_threadv313(), + py_isv312(), + py_runtimev313(), + py_gc(), + py_cframe(), + py_gilruntimestate(), +}; + // ----------------------------------------------------------------------------- const python_v* @@ -768,11 +808,14 @@ getCPythonOffsets(int major, int minor) case 12: return &python_v3_12; break; + case 13: + return &python_v3_13; + break; default: warnAboutUnsuportedVersion(major, minor); // fallthrough to latest - case 13: - return &python_v3_13; + case 14: + return &python_v3_14; break; } break; From dcbb0dbc5d94711975c353ea7c47eb15a372c0bc Mon Sep 17 00:00:00 2001 From: Matt Wozniski Date: Mon, 19 May 2025 23:01:56 -0400 Subject: [PATCH 02/13] ci: Avoid publishing 3.14 wheels We don't want to publish any wheels for Python 3.14 betas, since ABI stability isn't guaranteed until the release candidates. Signed-off-by: Matt Wozniski --- .github/workflows/build_wheels.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/build_wheels.yml b/.github/workflows/build_wheels.yml index e3aa9f74..abc4a527 100644 --- a/.github/workflows/build_wheels.yml +++ b/.github/workflows/build_wheels.yml @@ -353,6 +353,7 @@ jobs: mv dist/sdist/*.tar.gz dist/ mv dist/*-wheels/*.whl dist/ rmdir dist/{sdist,*-wheels} + rm -f dist/*cp314* ls -R dist - uses: pypa/gh-action-pypi-publish@release/v1 with: From b1f93931230011d7d927f15dac397c578c0a7261 Mon Sep 17 00:00:00 2001 From: Matt Wozniski Date: Fri, 30 May 2025 18:50:33 -0400 Subject: [PATCH 03/13] Add new fields from 3.14b2 to debug offsets Signed-off-by: Matt Wozniski --- src/pystack/_pystack/cpython/runtime.h | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/src/pystack/_pystack/cpython/runtime.h b/src/pystack/_pystack/cpython/runtime.h index 2d9adf86..5fa637bd 100644 --- a/src/pystack/_pystack/cpython/runtime.h +++ b/src/pystack/_pystack/cpython/runtime.h @@ -455,6 +455,8 @@ typedef struct _Py_DebugOffsets uint64_t gil_runtime_state_enabled; uint64_t gil_runtime_state_locked; uint64_t gil_runtime_state_holder; + uint64_t code_object_generation; + uint64_t tlbc_generation; } interpreter_state; // Thread state offset; @@ -597,6 +599,12 @@ typedef struct _Py_DebugOffsets uint64_t gi_frame_state; } gen_object; + struct _llist_node + { + uint64_t next; + uint64_t prev; + } llist_node; + struct _debugger_support { uint64_t eval_breaker; From 47144f66e6a81b8a7437bd0e58d829ca68df7fce Mon Sep 17 00:00:00 2001 From: Matt Wozniski Date: Fri, 30 May 2025 19:04:51 -0400 Subject: [PATCH 04/13] Ignore the low bits of PyStackRef when reading locals Signed-off-by: Matt Wozniski --- src/pystack/_pystack/pyframe.cpp | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/pystack/_pystack/pyframe.cpp b/src/pystack/_pystack/pyframe.cpp index e26e7d7b..3b81bebe 100644 --- a/src/pystack/_pystack/pyframe.cpp +++ b/src/pystack/_pystack/pyframe.cpp @@ -134,6 +134,12 @@ FrameObject::resolveLocalVariables() return; } + if (d_manager->versionIsAtLeast(3, 14)) { + // In Python 3.14, the local variable is a PyStackRef: a pointer + // with extra flags set in its low bits. Ignore the flags. + addr = addr & (~3); + } + std::string key = d_code->Varnames()[index]; LOG(DEBUG) << "Copying local variable at address " << std::hex << std::showbase << addr; From 15ef430e36d0bd582f232520a88df2a4a9b583be Mon Sep 17 00:00:00 2001 From: Matt Wozniski Date: Fri, 30 May 2025 19:42:19 -0400 Subject: [PATCH 05/13] Adjust a test to cope with 3.14 tuple changes PyTupleObject now includes a cached hash. Adjust our heuristics about how to find the items inside a tuple. Signed-off-by: Matt Wozniski --- tests/integration/test_local_variables.py | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/tests/integration/test_local_variables.py b/tests/integration/test_local_variables.py index d992eaee..341bfe1e 100644 --- a/tests/integration/test_local_variables.py +++ b/tests/integration/test_local_variables.py @@ -582,6 +582,10 @@ class TupleObject(ctypes.Structure): _fields_ = [ ("ob_type", ctypes.c_void_p), ("ob_size", ctypes.c_ssize_t), + ] + +class TupleItems(ctypes.Structure): + _fields_ = [ ("ob_item0", ctypes.c_void_p), ("ob_item1", ctypes.c_void_p), ] @@ -590,6 +594,10 @@ def ob_type_field(obj): # Assume ob_type is the last field of PyObject return id(obj) + sys.getsizeof(None) - ctypes.sizeof(ctypes.c_void_p) +def tuple_ob_items_field(tup): + assert isinstance(tup, tuple) + return id(tup) + sys.getsizeof(()) - sys.getsizeof(None) + def main(): bad_type = (1, 2, 3) bad_elem = (4, 5, 6) @@ -597,8 +605,8 @@ def main(): bad_list = [0, 1, 2] TupleObject.from_address(ob_type_field(bad_type)).ob_type = 0xded - TupleObject.from_address(ob_type_field(bad_elem)).ob_item1 = 0xbad - TupleObject.from_address(ob_type_field(nullelem)).ob_item1 = 0x0 + TupleItems.from_address(tuple_ob_items_field(bad_elem)).ob_item1 = 0xbad + TupleItems.from_address(tuple_ob_items_field(nullelem)).ob_item1 = 0x0 ListObject.from_address(ob_type_field(bad_list)).ob_item = 0x0 fifo = sys.argv[1] From 62f2d53dc4e243bf11e8025bdde0b346afcc74f9 Mon Sep 17 00:00:00 2001 From: Matt Wozniski Date: Fri, 30 May 2025 19:42:19 -0400 Subject: [PATCH 06/13] Adjust a test to cope with 3.14 tuple changes PyTupleObject now includes a cached hash. Adjust our heuristics about how to find the items inside a tuple. Signed-off-by: Matt Wozniski --- tests/integration/test_local_variables.py | 36 ++++++++++++----------- 1 file changed, 19 insertions(+), 17 deletions(-) diff --git a/tests/integration/test_local_variables.py b/tests/integration/test_local_variables.py index 341bfe1e..2156a78b 100644 --- a/tests/integration/test_local_variables.py +++ b/tests/integration/test_local_variables.py @@ -578,26 +578,28 @@ class ListObject(ctypes.Structure): ("ob_item", ctypes.c_void_p), ] -class TupleObject(ctypes.Structure): - _fields_ = [ - ("ob_type", ctypes.c_void_p), - ("ob_size", ctypes.c_ssize_t), - ] - -class TupleItems(ctypes.Structure): - _fields_ = [ - ("ob_item0", ctypes.c_void_p), - ("ob_item1", ctypes.c_void_p), - ] +if sys.version_info >= (3, 14): + class TupleObject(ctypes.Structure): + _fields_ = [ + ("ob_type", ctypes.c_void_p), + ("ob_size", ctypes.c_ssize_t), + ("ob_hash", ctypes.c_ssize_t), + ("ob_item0", ctypes.c_void_p), + ("ob_item1", ctypes.c_void_p), + ] +else: + class TupleObject(ctypes.Structure): + _fields_ = [ + ("ob_type", ctypes.c_void_p), + ("ob_size", ctypes.c_ssize_t), + ("ob_item0", ctypes.c_void_p), + ("ob_item1", ctypes.c_void_p), + ] def ob_type_field(obj): # Assume ob_type is the last field of PyObject return id(obj) + sys.getsizeof(None) - ctypes.sizeof(ctypes.c_void_p) -def tuple_ob_items_field(tup): - assert isinstance(tup, tuple) - return id(tup) + sys.getsizeof(()) - sys.getsizeof(None) - def main(): bad_type = (1, 2, 3) bad_elem = (4, 5, 6) @@ -605,8 +607,8 @@ def main(): bad_list = [0, 1, 2] TupleObject.from_address(ob_type_field(bad_type)).ob_type = 0xded - TupleItems.from_address(tuple_ob_items_field(bad_elem)).ob_item1 = 0xbad - TupleItems.from_address(tuple_ob_items_field(nullelem)).ob_item1 = 0x0 + TupleObject.from_address(ob_type_field(bad_elem)).ob_item1 = 0xbad + TupleObject.from_address(ob_type_field(nullelem)).ob_item1 = 0x0 ListObject.from_address(ob_type_field(bad_list)).ob_item = 0x0 fifo = sys.argv[1] From 5dced7f4e8ec6881efa312a1812d81d38bca800d Mon Sep 17 00:00:00 2001 From: Matt Wozniski Date: Sun, 1 Jun 2025 13:14:17 -0400 Subject: [PATCH 07/13] Handle PyUnicode object state layout changes in 3.14t Signed-off-by: Matt Wozniski --- src/pystack/_pystack/cpython/string.h | 18 ++++++++++++++++++ src/pystack/_pystack/process.cpp | 12 +++++++++--- src/pystack/_pystack/version.h | 2 +- 3 files changed, 28 insertions(+), 4 deletions(-) diff --git a/src/pystack/_pystack/cpython/string.h b/src/pystack/_pystack/cpython/string.h index cf61ddd8..b6618c3c 100644 --- a/src/pystack/_pystack/cpython/string.h +++ b/src/pystack/_pystack/cpython/string.h @@ -109,4 +109,22 @@ typedef struct } // namespace Python3_12 +namespace Python3_14t { + +struct _PyUnicode_State +{ + unsigned char interned; + unsigned int kind : 3; + unsigned int compact : 1; + unsigned int ascii : 1; + unsigned int statically_allocated : 1; +}; + +} // namespace Python3_14t + +union AnyPyUnicodeState { + Python3::_PyUnicode_State python3; + Python3_14t::_PyUnicode_State python3_14t; +}; + } // namespace pystack diff --git a/src/pystack/_pystack/process.cpp b/src/pystack/_pystack/process.cpp index b48a2252..52ec0099 100644 --- a/src/pystack/_pystack/process.cpp +++ b/src/pystack/_pystack/process.cpp @@ -540,9 +540,15 @@ AbstractProcessManager::getStringFromAddress(remote_addr_t addr) const << addr; Structure unicode(shared_from_this(), addr); - Python3::_PyUnicode_State state = unicode.getField(&py_unicode_v::o_state); - if (state.kind != 1 || state.compact != 1) { - throw InvalidRemoteObject(); + AnyPyUnicodeState state = unicode.getField(&py_unicode_v::o_state); + if (versionIsAtLeast(3, 14) and isFreeThreaded()) { + if (state.python3_14t.kind != 1 || state.python3_14t.compact != 1) { + throw InvalidRemoteObject(); + } + } else { + if (state.python3.kind != 1 || state.python3.compact != 1) { + throw InvalidRemoteObject(); + } } len = unicode.getField(&py_unicode_v::o_length); diff --git a/src/pystack/_pystack/version.h b/src/pystack/_pystack/version.h index ae532ce2..c56851ac 100644 --- a/src/pystack/_pystack/version.h +++ b/src/pystack/_pystack/version.h @@ -77,7 +77,7 @@ struct py_bytes_v struct py_unicode_v { ssize_t size; - FieldOffset o_state; + FieldOffset o_state; FieldOffset o_length; FieldOffset o_ascii; }; From 4d8198dbd7093de8f109c297cc30f5dd69280a9f Mon Sep 17 00:00:00 2001 From: Matt Wozniski Date: Sun, 1 Jun 2025 13:14:40 -0400 Subject: [PATCH 08/13] Avoid trying to read the code for shim frames Signed-off-by: Matt Wozniski --- src/pystack/_pystack/pycode.cpp | 1 + src/pystack/_pystack/pyframe.cpp | 4 ++-- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/src/pystack/_pystack/pycode.cpp b/src/pystack/_pystack/pycode.cpp index fc490e37..874fa255 100644 --- a/src/pystack/_pystack/pycode.cpp +++ b/src/pystack/_pystack/pycode.cpp @@ -133,6 +133,7 @@ getLocationInfo( tlbc_entries + sizeof(tlbc_size), tlbc_size * sizeof(uintptr_t), vec.data()); + LOG(DEBUG) << "tlbc_index=" << tlbc_index << " tlbc_size=" << tlbc_size; uintptr_t code_adaptive_actual = vec[tlbc_index]; ptrdiff_t addrq = (reinterpret_cast(last_instruction_index) diff --git a/src/pystack/_pystack/pyframe.cpp b/src/pystack/_pystack/pyframe.cpp index 3b81bebe..62769529 100644 --- a/src/pystack/_pystack/pyframe.cpp +++ b/src/pystack/_pystack/pyframe.cpp @@ -29,10 +29,10 @@ FrameObject::FrameObject( if (d_is_shim) { LOG(DEBUG) << "Skipping over a shim frame inserted by the interpreter"; next_frame_no = frame_no; + } else { + d_code = getCode(manager, frame); } - d_code = getCode(manager, frame); - auto prev_addr = frame.getField(&py_frame_v::o_back); LOG(DEBUG) << std::hex << std::showbase << "Previous frame address: " << prev_addr; if (prev_addr) { From 9e80b23e971984cb138bed92f54bfef99d3934e5 Mon Sep 17 00:00:00 2001 From: Matt Wozniski Date: Tue, 15 Jul 2025 17:40:36 -0400 Subject: [PATCH 09/13] Handle 3.14 version string's free-threading marker In 3.14.0b3 this string was changed to "free-threading build" from "experimental free-threading build". Signed-off-by: Matt Wozniski --- src/pystack/process.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/pystack/process.py b/src/pystack/process.py index 75e93c78..c44407f9 100644 --- a/src/pystack/process.py +++ b/src/pystack/process.py @@ -24,7 +24,7 @@ # or "3.13.0+ experimental free-threading build (Python)" BSS_VERSION_REGEXP = re.compile( rb"((2|3)\.(\d+)\.(\d{1,2}))((a|b|c|rc)\d{1,2})?\+?" - rb"(?: experimental free-threading build)? (\(.{1,64}\))" + rb"(?: (?:experimental )?free-threading build)? (\(.{1,64}\))" ) LOGGER = logging.getLogger(__file__) From 814bc1c5221a31c5f33d19d390811155d4aadd7d Mon Sep 17 00:00:00 2001 From: Matt Wozniski Date: Thu, 14 Aug 2025 19:28:11 -0400 Subject: [PATCH 10/13] test: Discover and test with 3.14 and 3.14t Signed-off-by: Matt Wozniski --- tests/utils.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/utils.py b/tests/utils.py index b2321400..dacb18fa 100644 --- a/tests/utils.py +++ b/tests/utils.py @@ -23,6 +23,8 @@ PythonVersion = Tuple[Tuple[int, int], pathlib.Path] ALL_VERSIONS = [ + ((3, 14), "python3.14t"), + ((3, 14), "python3.14"), ((3, 13), "python3.13t"), ((3, 13), "python3.13"), ((3, 12), "python3.12"), From 658acf703b986e0d84c146074cc2127f96ff5887 Mon Sep 17 00:00:00 2001 From: Matt Wozniski Date: Thu, 14 Aug 2025 19:52:25 -0400 Subject: [PATCH 11/13] Support free-threading builds of Python 3.14t Build wheels for 3.14t, and declare that our extension module doesn't rely on the GIL. Signed-off-by: Matt Wozniski --- .github/workflows/build_wheels.yml | 2 +- setup.py | 2 ++ tests/integration/empty_thread_extension/testext.cpp | 6 +++++- .../empty_thread_extension_with_os_threads/testext.cpp | 6 +++++- 4 files changed, 13 insertions(+), 3 deletions(-) diff --git a/.github/workflows/build_wheels.yml b/.github/workflows/build_wheels.yml index abc4a527..539a268b 100644 --- a/.github/workflows/build_wheels.yml +++ b/.github/workflows/build_wheels.yml @@ -68,7 +68,7 @@ jobs: - name: Build wheels uses: pypa/cibuildwheel@v3.0.1 env: - CIBW_BUILD: "cp3{8..14}-${{ matrix.wheel_type }}" + CIBW_BUILD: "cp3{8..14}{t,}-${{ matrix.wheel_type }}" CIBW_ARCHS_LINUX: auto aarch64 CIBW_ENABLE: cpython-prerelease - uses: actions/upload-artifact@v4 diff --git a/setup.py b/setup.py index 956f55aa..477bb977 100644 --- a/setup.py +++ b/setup.py @@ -33,6 +33,7 @@ "cdivision": True, "c_string_type": "unicode", "c_string_encoding": "utf8", + "freethreading_compatible": True, } DEFINE_MACROS = [] @@ -50,6 +51,7 @@ "infer_types": True, "c_string_type": "unicode", "c_string_encoding": "utf8", + "freethreading_compatible": True, } DEFINE_MACROS.extend([("CYTHON_TRACE", "1"), ("CYTHON_TRACE_NOGIL", "1")]) diff --git a/tests/integration/empty_thread_extension/testext.cpp b/tests/integration/empty_thread_extension/testext.cpp index a01868f7..faebd2b8 100644 --- a/tests/integration/empty_thread_extension/testext.cpp +++ b/tests/integration/empty_thread_extension/testext.cpp @@ -39,7 +39,11 @@ static struct PyModuleDef moduledef = {PyModuleDef_HEAD_INIT, "testext", "", -1, PyMODINIT_FUNC PyInit_testext(void) { - return PyModule_Create(&moduledef); + PyObject* mod = PyModule_Create(&moduledef); +# ifdef Py_GIL_DISABLED + PyUnstable_Module_SetGIL(mod, Py_MOD_GIL_NOT_USED); +# endif + return mod; } #else PyMODINIT_FUNC diff --git a/tests/integration/empty_thread_extension_with_os_threads/testext.cpp b/tests/integration/empty_thread_extension_with_os_threads/testext.cpp index a552646d..cc12fd32 100644 --- a/tests/integration/empty_thread_extension_with_os_threads/testext.cpp +++ b/tests/integration/empty_thread_extension_with_os_threads/testext.cpp @@ -69,7 +69,11 @@ static struct PyModuleDef moduledef = {PyModuleDef_HEAD_INIT, "testext", "", -1, PyMODINIT_FUNC PyInit_testext(void) { - return PyModule_Create(&moduledef); + PyObject* mod = PyModule_Create(&moduledef); +# ifdef Py_GIL_DISABLED + PyUnstable_Module_SetGIL(mod, Py_MOD_GIL_NOT_USED); +# endif + return mod; } #else PyMODINIT_FUNC From 6740edc605bb94481a6acacb793f92d604e6a6d9 Mon Sep 17 00:00:00 2001 From: Matt Wozniski Date: Thu, 14 Aug 2025 19:59:44 -0400 Subject: [PATCH 12/13] ci: Allow publishing 3.14 wheels This reverts commit dcbb0dbc5d94711975c353ea7c47eb15a372c0bc. We're now in the release candidates, which have a stable ABI. Signed-off-by: Matt Wozniski --- .github/workflows/build_wheels.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/build_wheels.yml b/.github/workflows/build_wheels.yml index 539a268b..ba4f0eca 100644 --- a/.github/workflows/build_wheels.yml +++ b/.github/workflows/build_wheels.yml @@ -353,7 +353,6 @@ jobs: mv dist/sdist/*.tar.gz dist/ mv dist/*-wheels/*.whl dist/ rmdir dist/{sdist,*-wheels} - rm -f dist/*cp314* ls -R dist - uses: pypa/gh-action-pypi-publish@release/v1 with: From 4385c40c35aa087ad9d72b3993f8fc1ce13a8acd Mon Sep 17 00:00:00 2001 From: Matt Wozniski Date: Thu, 14 Aug 2025 20:11:40 -0400 Subject: [PATCH 13/13] ci: Stop installing the `python3-distutils` dpkg We're creating a virtual environment and installing `setuptools` into it, and that's all we actually need. Signed-off-by: Matt Wozniski --- .github/workflows/build_wheels.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/build_wheels.yml b/.github/workflows/build_wheels.yml index ba4f0eca..0c74c15e 100644 --- a/.github/workflows/build_wheels.yml +++ b/.github/workflows/build_wheels.yml @@ -323,8 +323,7 @@ jobs: python3-dev \ python3-pip \ python3-venv \ - python3-dbg \ - python3-distutils + python3-dbg - uses: actions/download-artifact@v4 with: name: "manylinux_x86_64-wheels"