From 1de25fa770dc17708e2aa289bd1bebb8370d3acb Mon Sep 17 00:00:00 2001 From: Tiago Medicci Date: Fri, 24 Jan 2025 09:02:57 -0300 Subject: [PATCH 1/4] interpreters/python: add patch to set `_PyRuntime` section By setting a specific region for the `_PyRuntime` structure, it is possible to move it to the external memory, for instance, freeing the internal memory (this structure occupies around 140KiB). --- interpreters/python/Makefile | 1 + ...ntime-structure-into-PSRAM-bss-regio.patch | 54 +++++++++++++++++++ 2 files changed, 55 insertions(+) create mode 100644 interpreters/python/patch/0012-hack-place-_PyRuntime-structure-into-PSRAM-bss-regio.patch diff --git a/interpreters/python/Makefile b/interpreters/python/Makefile index 4b6c42eac39..c478ec1e420 100644 --- a/interpreters/python/Makefile +++ b/interpreters/python/Makefile @@ -80,6 +80,7 @@ $(CPYTHON_UNPACKNAME): $(CPYTHON_ZIP) $(Q) patch -p1 -d $(CPYTHON_UNPACKNAME) < patch$(DELIM)0009-include-nuttx-sys-select-header-to-define-FD_SETSIZE.patch $(Q) patch -p1 -d $(CPYTHON_UNPACKNAME) < patch$(DELIM)0010-check-for-the-d_ino-member-of-the-structure-dirent.patch $(Q) patch -p1 -d $(CPYTHON_UNPACKNAME) < patch$(DELIM)0011-avoid-redefinition-warning-if-UNUSED-is-already-defi.patch + $(Q) patch -p1 -d $(CPYTHON_UNPACKNAME) < patch$(DELIM)0012-hack-place-_PyRuntime-structure-into-PSRAM-bss-regio.patch $(HOSTPYTHON): mkdir -p $(HOSTBUILD) diff --git a/interpreters/python/patch/0012-hack-place-_PyRuntime-structure-into-PSRAM-bss-regio.patch b/interpreters/python/patch/0012-hack-place-_PyRuntime-structure-into-PSRAM-bss-regio.patch new file mode 100644 index 00000000000..d64342e551c --- /dev/null +++ b/interpreters/python/patch/0012-hack-place-_PyRuntime-structure-into-PSRAM-bss-regio.patch @@ -0,0 +1,54 @@ +From d1e903f516849c535455904b3c3f8a33665c1a88 Mon Sep 17 00:00:00 2001 +From: Ivan Grokhotkov +Date: Wed, 23 Oct 2024 16:52:52 +0200 +Subject: [PATCH 12/12] hack: place _PyRuntime structure into PSRAM bss region, + initialize later + +_PyRuntime occupies around 100kB of RAM in .data region, making it +hard to fit the interpreter into the available static RAM. +This patch moves it into PSRAM using section attribute. Normally +we shouldn't need this as we can specify placements in ldfragments, +however in this specific case I couldn't get it to work. +Since the structure is now in .bss, add a function which will assign +it the initial value. + +The proper fix might be to support .data segment on PSRAM in IDF, +as well as to fix whatever ldgen issue prevents this variable from +being moved to PSRAM. + +Co-authored-by: Tiago Medicci Serrano +--- + Python/pylifecycle.c | 11 +++++++++++ + 1 file changed, 11 insertions(+) + +diff --git a/Python/pylifecycle.c b/Python/pylifecycle.c +index 1701a1cd217..2a8e544f0ac 100644 +--- a/Python/pylifecycle.c ++++ b/Python/pylifecycle.c +@@ -102,12 +102,23 @@ __attribute__(( + _PyRuntimeState _PyRuntime + #if defined(__linux__) && (defined(__GNUC__) || defined(__clang__)) + __attribute__ ((section (".PyRuntime"))) ++#elif defined(ESP_PLATFORM) ++__attribute__ ((section (".PyRuntime"))) + #endif + = _PyRuntimeState_INIT(_PyRuntime, _Py_Debug_Cookie); + _Py_COMP_DIAG_POP + + static int runtime_initialized = 0; + ++void _PyRuntime_Early_Init(void) { ++#if defined(ESP_PLATFORM) ++ // Normally, _PyRuntime is in .data and is initialized by the C runtime. ++ // This function allows us to place it into external RAM .bss section ++ // and initialize it manually, saving some internal RAM. ++ _PyRuntime = (struct pyruntimestate) _PyRuntimeState_INIT(_PyRuntime, _Py_Debug_Cookie); ++#endif ++} ++ + PyStatus + _PyRuntime_Initialize(void) + { +-- +2.47.1 + From 7009201812e338172edd951015fbb4130f68e18e Mon Sep 17 00:00:00 2001 From: Tiago Medicci Date: Tue, 21 Jan 2025 17:32:37 -0300 Subject: [PATCH 2/4] interpreters/python: create Python's config files dynamically The `Setup.local` and the `config.site` files are used by Python's build system to, respectively, enable or disable Python's modules and set/unset available functions in the target system. These files are now set according to NuttX's configs, enabling or disabling Python's features according to the configs set on NuttX. --- interpreters/python/.gitignore | 2 ++ interpreters/python/Makefile | 34 ++++++++++++++++++- .../python/{Setup.local => Setup.local.in} | 0 .../python/{config.site => config.site.in} | 4 +-- 4 files changed, 37 insertions(+), 3 deletions(-) rename interpreters/python/{Setup.local => Setup.local.in} (100%) rename interpreters/python/{config.site => config.site.in} (95%) diff --git a/interpreters/python/.gitignore b/interpreters/python/.gitignore index ad58719cdd4..33efb9cbe30 100644 --- a/interpreters/python/.gitignore +++ b/interpreters/python/.gitignore @@ -4,3 +4,5 @@ /Python/ /romfs_cpython_modules.h /romfs_cpython_modules.img +/config.site +/Setup.local diff --git a/interpreters/python/Makefile b/interpreters/python/Makefile index c478ec1e420..810904f502b 100644 --- a/interpreters/python/Makefile +++ b/interpreters/python/Makefile @@ -32,6 +32,7 @@ UNPACK ?= unzip -q -o MACHDEP=nuttx CONFIG_SITE=${CURDIR}/config.site +SETUP_LOCAL=${CURDIR}/Setup.local CPYTHON_PATH=$(CURDIR)/$(CPYTHON_UNPACKNAME) BUILDIR=$(CURDIR)/build @@ -92,6 +93,35 @@ $(HOSTPYTHON): ) $(MAKE) -C $(HOSTBUILD) install +# The `config.site` file contains settings that override the configuration +# settings provided by the `configure` script. Depending on the features +# enabled on NuttX, this file may need to be adjusted. + +$(CONFIG_SITE): + $(Q) ( cp $(CONFIG_SITE).in $(CONFIG_SITE)) +ifeq ($(CONFIG_ARCH_HAVE_FORK),y) + @echo "export ac_cv_func_fork=\"yes\"" >> $@ +else + @echo "export ac_cv_func_fork=\"no\"" >> $@ +endif +ifeq ($(CONFIG_SYSTEM_SYSTEM),y) + @echo "export ac_cv_func_system=\"yes\"" >> $@ +else + @echo "export ac_cv_func_system=\"no\"" >> $@ +endif + +# The `Setup.local` file enables or disables Python modules. +# Depending on the features enabled on NuttX, this file may need to be +# adjusted. Please note that the base `Setup.local.in` file only contains +# a section to disable Python modules. Inserting lines to it will disable +# such modules. + +$(SETUP_LOCAL): + $(Q) ( cp $(SETUP_LOCAL).in $(SETUP_LOCAL)) +ifneq ($(CONFIG_ARCH_HAVE_FORK),y) + @echo "_posixsubprocess" >> $@ +endif + # For the Python's `configure` script, please consider the following # when building for NuttX: # @@ -103,7 +133,7 @@ $(HOSTPYTHON): # Python/Modules/getpath.c (issue will be filed soon to track this # problem). -$(TARGETBUILD)/Makefile: $(HOSTPYTHON) +$(TARGETBUILD)/Makefile: $(HOSTPYTHON) $(CONFIG_SITE) $(SETUP_LOCAL) $(Q) mkdir -p $(TARGETBUILD)/Modules $(Q) mkdir -p $(TARGETMODULES)/python$(CPYTHON_VERSION_MINOR) $(Q) ( cp Setup.local $(TARGETBUILD)/Modules/Setup.local ) @@ -177,5 +207,7 @@ distclean:: $(call DELFILE, $(CPYTHON_ZIP)) $(call DELFILE, romfs_cpython_modules.img) $(call DELFILE, romfs_cpython_modules.h) + $(call DELFILE, config.site) + $(call DELFILE, Setup.local) include $(APPDIR)/Application.mk diff --git a/interpreters/python/Setup.local b/interpreters/python/Setup.local.in similarity index 100% rename from interpreters/python/Setup.local rename to interpreters/python/Setup.local.in diff --git a/interpreters/python/config.site b/interpreters/python/config.site.in similarity index 95% rename from interpreters/python/config.site rename to interpreters/python/config.site.in index 28313d75e7e..eb37e5a88c9 100644 --- a/interpreters/python/config.site +++ b/interpreters/python/config.site.in @@ -1,3 +1,4 @@ +export MODULE_BUILDTYPE="static" export ac_cv_file__dev_ptmx="no" export ac_cv_file__dev_ptc="no" export ac_cv_buggy_getaddrinfo="no" @@ -15,10 +16,9 @@ export ac_cv_func_clock_gettime="yes" export ac_cv_header_sys_syscall_h="no" export ac_cv_func_timegm="yes" export ac_cv_func_clock="yes" -export ac_cv_func_fork="yes" export ac_cv_func_waitpid="yes" export ac_cv_func_pipe="yes" export ac_cv_enable_strict_prototypes_warning="no" export ac_cv_func_getnameinfo="yes" export ac_cv_func_poll="yes" -export MODULE_BUILDTYPE="static" +export ac_cv_func_gethostname="yes" \ No newline at end of file From f3bcfc1e5c04f371c9defc8ba5237745ed10c30f Mon Sep 17 00:00:00 2001 From: Tiago Medicci Date: Wed, 29 Jan 2025 17:10:49 -0300 Subject: [PATCH 3/4] interpreters/python: add wrapper to initialize Python This wrapper application checks if the Python's modules are already mounted (and mounts them, if not), sets the necessary environment variables and, then, runs the Python interpreter. --- interpreters/python/Kconfig | 20 +--- interpreters/python/Makefile | 9 +- ...functions-used-by-NuttX-to-lowercase.patch | 33 +++++++ .../{mount_modules.c => python_wrapper.c} | 92 ++++++++++++++++--- 4 files changed, 116 insertions(+), 38 deletions(-) create mode 100644 interpreters/python/patch/0013-transform-functions-used-by-NuttX-to-lowercase.patch rename interpreters/python/{mount_modules.c => python_wrapper.c} (72%) diff --git a/interpreters/python/Kconfig b/interpreters/python/Kconfig index da20c8ec43f..947a71e6de2 100644 --- a/interpreters/python/Kconfig +++ b/interpreters/python/Kconfig @@ -27,7 +27,7 @@ config INTERPRETER_CPYTHON_STACKSIZE config INTERPRETER_CPYTHON_PRIORITY int "CPython task priority" - default 150 + default 100 ---help--- This is the priority of the CPython task. @@ -37,22 +37,4 @@ config INTERPRETER_CPYTHON_PROGNAME ---help--- This is the name of the program that will be used from the nsh. -config INTERPRETER_CPYTHON_MOUNT_MODULES_STACKSIZE - int "CPython's Modules Mount stack size" - default 4096 - ---help--- - This is the stack size allocated when the CPython's Modules Mount task runs. - -config INTERPRETER_CPYTHON_MOUNT_MODULES_PRIORITY - int "CPython's Modules Mount task priority" - default 150 - ---help--- - This is the priority of the CPython's Modules Mount task. - -config INTERPRETER_CPYTHON_MOUNT_MODULES_PROGNAME - string "CPython's Modules Mount app name" - default "python_mount_modules" - ---help--- - This is the name of the program that will be used from the nsh. - endif diff --git a/interpreters/python/Makefile b/interpreters/python/Makefile index 810904f502b..b6130af5f74 100644 --- a/interpreters/python/Makefile +++ b/interpreters/python/Makefile @@ -82,6 +82,7 @@ $(CPYTHON_UNPACKNAME): $(CPYTHON_ZIP) $(Q) patch -p1 -d $(CPYTHON_UNPACKNAME) < patch$(DELIM)0010-check-for-the-d_ino-member-of-the-structure-dirent.patch $(Q) patch -p1 -d $(CPYTHON_UNPACKNAME) < patch$(DELIM)0011-avoid-redefinition-warning-if-UNUSED-is-already-defi.patch $(Q) patch -p1 -d $(CPYTHON_UNPACKNAME) < patch$(DELIM)0012-hack-place-_PyRuntime-structure-into-PSRAM-bss-regio.patch + $(Q) patch -p1 -d $(CPYTHON_UNPACKNAME) < patch$(DELIM)0013-transform-functions-used-by-NuttX-to-lowercase.patch $(HOSTPYTHON): mkdir -p $(HOSTBUILD) @@ -175,13 +176,7 @@ PROGNAME += $(CONFIG_INTERPRETER_CPYTHON_PROGNAME) PRIORITY += $(CONFIG_INTERPRETER_CPYTHON_PRIORITY) STACKSIZE += $(CONFIG_INTERPRETER_CPYTHON_STACKSIZE) -MAINSRC += python.c - -PROGNAME += $(CONFIG_INTERPRETER_CPYTHON_MOUNT_MODULES_PROGNAME) -PRIORITY += $(CONFIG_INTERPRETER_CPYTHON_MOUNT_MODULES_PRIORITY) -STACKSIZE += $(CONFIG_INTERPRETER_CPYTHON_MOUNT_MODULES_STACKSIZE) - -MAINSRC += mount_modules.c +MAINSRC += python_wrapper.c checkgenromfs: @genromfs -h 1>/dev/null 2>&1 || { \ diff --git a/interpreters/python/patch/0013-transform-functions-used-by-NuttX-to-lowercase.patch b/interpreters/python/patch/0013-transform-functions-used-by-NuttX-to-lowercase.patch new file mode 100644 index 00000000000..c1e83aad640 --- /dev/null +++ b/interpreters/python/patch/0013-transform-functions-used-by-NuttX-to-lowercase.patch @@ -0,0 +1,33 @@ +From 914c80b7969d73840bc1b573b478d9148999b7d0 Mon Sep 17 00:00:00 2001 +From: Tiago Medicci +Date: Fri, 31 Jan 2025 14:06:21 -0300 +Subject: [PATCH 13/13] transform functions used by NuttX to lowercase + +--- + Include/pylifecycle.h | 10 ++++++++++ + 1 file changed, 10 insertions(+) + +diff --git a/Include/pylifecycle.h b/Include/pylifecycle.h +index de1bcb1d2cb..044780ee188 100644 +--- a/Include/pylifecycle.h ++++ b/Include/pylifecycle.h +@@ -33,6 +33,16 @@ PyAPI_FUNC(void) _Py_NO_RETURN Py_Exit(int); + PyAPI_FUNC(int) Py_Main(int argc, wchar_t **argv); + PyAPI_FUNC(int) Py_BytesMain(int argc, char **argv); + ++#if defined(__NuttX__) ++#define py_bytesmain Py_BytesMain ++#endif ++ ++void _PyRuntime_Early_Init(void); ++ ++#if defined(__NuttX__) ++#define _pyruntime_early_init _PyRuntime_Early_Init ++#endif ++ + /* In pathconfig.c */ + Py_DEPRECATED(3.11) PyAPI_FUNC(void) Py_SetProgramName(const wchar_t *); + Py_DEPRECATED(3.13) PyAPI_FUNC(wchar_t *) Py_GetProgramName(void); +-- +2.47.1 + diff --git a/interpreters/python/mount_modules.c b/interpreters/python/python_wrapper.c similarity index 72% rename from interpreters/python/mount_modules.c rename to interpreters/python/python_wrapper.c index 681031579c4..1eb1d3cc3e0 100644 --- a/interpreters/python/mount_modules.c +++ b/interpreters/python/python_wrapper.c @@ -1,5 +1,5 @@ /**************************************************************************** - * apps/interpreters/python/mount_modules.c + * apps/interpreters/python/python_wrapper.c * * SPDX-License-Identifier: Apache-2.0 * @@ -41,11 +41,14 @@ #include #include #include +#include #include #include "romfs_cpython_modules.h" +#include "Python.h" + /**************************************************************************** * Pre-processor Definitions ****************************************************************************/ @@ -61,7 +64,7 @@ #endif #ifndef CONFIG_CPYTHON_ROMFS_MOUNTPOINT -# define CONFIG_CPYTHON_ROMFS_MOUNTPOINT "/usr/local/lib/" +# define CONFIG_CPYTHON_ROMFS_MOUNTPOINT "/usr/local/lib" #endif #ifdef CONFIG_DISABLE_MOUNTPOINT @@ -90,19 +93,57 @@ ****************************************************************************/ /**************************************************************************** - * Public Functions - ****************************************************************************/ - -/**************************************************************************** - * Name: mount_modules + * Name: check_and_mount_romfs + * + * Description: + * Check if the ROMFS is already mounted, and if not, mount it. + * + * Input Parameters: + * None + * + * Returned Value: + * 0 on success, 1 on failure + * ****************************************************************************/ -int main(int argc, FAR char *argv[]) +static int check_and_mount_romfs(void) { - int ret; + int ret = OK; struct boardioc_romdisk_s desc; + FILE *fp; + char line[256]; + int is_mounted = 0; - /* Create a RAM disk for the test */ + /* Check if the device is already mounted */ + + fp = fopen("/proc/fs/mount", "r"); + if (fp == NULL) + { + printf("ERROR: Failed to open /proc/fs/mount\n"); + UNUSED(desc); + return ret = ERROR; + } + + while (fgets(line, sizeof(line), fp)) + { + if (strstr(line, CONFIG_CPYTHON_ROMFS_MOUNTPOINT) != NULL) + { + is_mounted = 1; + break; + } + } + + fclose(fp); + + if (is_mounted) + { + _info("Device is already mounted at %s\n", + CONFIG_CPYTHON_ROMFS_MOUNTPOINT); + UNUSED(desc); + return ret; + } + + /* Create a RAM disk */ desc.minor = CONFIG_CPYTHON_ROMFS_RAMDEVNO; /* Minor device number of the ROM disk. */ desc.nsectors = NSECTORS(romfs_cpython_modules_img_len); /* The number of sectors in the ROM disk */ @@ -119,8 +160,8 @@ int main(int argc, FAR char *argv[]) /* Mount the test file system */ - printf("Mounting ROMFS filesystem at target=%s with source=%s\n", - CONFIG_CPYTHON_ROMFS_MOUNTPOINT, MOUNT_DEVNAME); + _info("Mounting ROMFS filesystem at target=%s with source=%s\n", + CONFIG_CPYTHON_ROMFS_MOUNTPOINT, MOUNT_DEVNAME); ret = mount(MOUNT_DEVNAME, CONFIG_CPYTHON_ROMFS_MOUNTPOINT, "romfs", MS_RDONLY, NULL); @@ -132,3 +173,30 @@ int main(int argc, FAR char *argv[]) return 0; } + +/**************************************************************************** + * Public Functions + ****************************************************************************/ + +/**************************************************************************** + * Name: python_wrapper_main + ****************************************************************************/ + +int main(int argc, FAR char *argv[]) +{ + int ret; + + ret = check_and_mount_romfs(); + if (ret != 0) + { + return ret; + } + + _pyruntime_early_init(); + + setenv("PYTHONHOME", "/usr/local", 1); + + setenv("PYTHON_BASIC_REPL", "1", 1); + + return py_bytesmain(argc, argv); +} From 3e5448f05ec5a921f544aa61e3a54aca1b702dec Mon Sep 17 00:00:00 2001 From: Tiago Medicci Date: Thu, 23 Jan 2025 16:43:01 -0300 Subject: [PATCH 4/4] interpreters/python: set ROMFS-generated data to const char This allows the data to be placed in the .rodata section, which can be allocated in the flash or other read-only storage, freeing the internal memory. --- interpreters/python/Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/interpreters/python/Makefile b/interpreters/python/Makefile index b6130af5f74..af5c8b89f6a 100644 --- a/interpreters/python/Makefile +++ b/interpreters/python/Makefile @@ -189,7 +189,7 @@ romfs_cpython_modules.img : $(TARGETLIBPYTHON) checkgenromfs @genromfs -f $@ -d $(TARGETMODULES) -V "ROMFS_Test" || { echo "genromfs failed" ; exit 1 ; } romfs_cpython_modules.h : romfs_cpython_modules.img - @xxd -i $< >$@ || { echo "xxd of $< failed" ; exit 1 ; } + @xxd -i $< | sed -e "s/^unsigned/static const unsigned/g" >$@ || { echo "xxd of $< failed" ; exit 1 ; } context:: $(CPYTHON_UNPACKNAME)