From 54227cbf83527f5bf930e8bfa93adb0c36accfc0 Mon Sep 17 00:00:00 2001 From: Florian Felten Date: Tue, 14 Oct 2025 09:48:11 +0200 Subject: [PATCH 1/6] Add python 3.13 support --- .github/workflows/test.yml | 6 +++--- pyproject.toml | 3 ++- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 15cf5a99..0f11b54d 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -13,7 +13,7 @@ jobs: timeout-minutes: 30 strategy: matrix: - python-version: ['3.10', '3.11', '3.12'] + python-version: ['3.10', '3.11', '3.12', '3.13'] steps: - uses: actions/checkout@v4 - name: Set up Python ${{ matrix.python-version }} @@ -38,7 +38,7 @@ jobs: timeout-minutes: 30 strategy: matrix: - python-version: ['3.12'] + python-version: ['3.13'] steps: - uses: actions/checkout@v4 - name: Set up Python ${{ matrix.python-version }} @@ -72,7 +72,7 @@ jobs: timeout-minutes: 30 strategy: matrix: - python-version: ['3.12'] + python-version: ['3.13'] steps: - uses: actions/checkout@v4 - name: Set up Python ${{ matrix.python-version }} diff --git a/pyproject.toml b/pyproject.toml index ae65e0a3..0cf7c71f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -17,11 +17,12 @@ classifiers = [ "Programming Language :: Python :: 3.10", "Programming Language :: Python :: 3.11", "Programming Language :: Python :: 3.12", + "Programming Language :: Python :: 3.13", 'Intended Audience :: Science/Research', 'Topic :: Scientific/Engineering :: Artificial Intelligence', ] dependencies = [ - "numpy <=2.0", + "numpy", "gymnasium >= 1.0.0", "datasets[vision] >= 3.1.0", # imports modules with image features "pandas >= 2.2.3", From 5b0193c671f2a431eda5867688e981fba6c72645 Mon Sep 17 00:00:00 2001 From: Florian Felten Date: Tue, 14 Oct 2025 10:02:37 +0200 Subject: [PATCH 2/6] Update slurm doctest --- docs/utils/slurm.md | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/docs/utils/slurm.md b/docs/utils/slurm.md index 19d37a10..311fa9dc 100644 --- a/docs/utils/slurm.md +++ b/docs/utils/slurm.md @@ -78,18 +78,18 @@ print(reduced) ```{testoutput} slurm [array([[0., ..., 0.], - ..., - [0., ..., 0.]]), array([[0., ..., 0.], - ..., - [0., ..., 0.]]), array([[0., ..., 0.], - ..., - [0., ..., 0.]]), array([[0., ..., 0.], - ..., - [0., ..., 0.]]), array([[0., ..., 0.], - ..., - [0., ..., 0.]]), array([[0., ..., 0.], - ..., - [0., ..., 0.]])] + ..., + [0., ..., 0.]], shape=(100, 50)), array([[0., ..., 0.], + ..., + [0., ..., 0.]], shape=(100, 50)), array([[0., ..., 0.], + ..., + [0., ..., 0.]], shape=(100, 50)), array([[0., ..., 0.], + ..., + [0., ..., 0.]], shape=(100, 50)), array([[0., ..., 0.], + ..., + [0., ..., 0.]], shape=(100, 50)), array([[0., ..., 0.], + ..., + [0., ..., 0.]], shape=(100, 50))] ``` **Step 4b:** Save all results in one [pickle](https://docs.python.org/3/library/pickle.html) archive From 7bcbfc09577f7c4d7184a39f428c35026ea90900 Mon Sep 17 00:00:00 2001 From: Florian Felten Date: Tue, 14 Oct 2025 10:07:37 +0200 Subject: [PATCH 3/6] fix doctest check to be less picky on text output --- docs/utils/slurm.md | 16 ++-------------- 1 file changed, 2 insertions(+), 14 deletions(-) diff --git a/docs/utils/slurm.md b/docs/utils/slurm.md index 311fa9dc..2a5a57a0 100644 --- a/docs/utils/slurm.md +++ b/docs/utils/slurm.md @@ -73,23 +73,11 @@ reduced = job.reduce(list, slurm_args=slurm_args) # Collect all results to a lis # To save resources, to render the docs no actual optimization is performed. # Instead optimize() is replaced by a method returning zeros: -print(reduced) +print(reduced) # doctest: +ELLIPSIS ``` ```{testoutput} slurm -[array([[0., ..., 0.], - ..., - [0., ..., 0.]], shape=(100, 50)), array([[0., ..., 0.], - ..., - [0., ..., 0.]], shape=(100, 50)), array([[0., ..., 0.], - ..., - [0., ..., 0.]], shape=(100, 50)), array([[0., ..., 0.], - ..., - [0., ..., 0.]], shape=(100, 50)), array([[0., ..., 0.], - ..., - [0., ..., 0.]], shape=(100, 50)), array([[0., ..., 0.], - ..., - [0., ..., 0.]], shape=(100, 50))] +[array([[0., ..., 0.]], shape=(100, 50)), ...] ``` **Step 4b:** Save all results in one [pickle](https://docs.python.org/3/library/pickle.html) archive From 2e1c3228a5d52a1237743fe68b1e6211c17550e7 Mon Sep 17 00:00:00 2001 From: Florian Felten Date: Tue, 14 Oct 2025 10:23:43 +0200 Subject: [PATCH 4/6] Use invalid string instead of ellipsis for pickling the traceback --- tests/utils/test_slurm.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/utils/test_slurm.py b/tests/utils/test_slurm.py index 3ee24bc5..53290e99 100644 --- a/tests/utils/test_slurm.py +++ b/tests/utils/test_slurm.py @@ -165,7 +165,7 @@ def test_sbatch_map_reduce_with_exception(sbatch_exec: str) -> None: """Test if a fake slurm can process FakeProblem.""" slurm_args = slurm.SlurmConfig(sbatch_executable=sbatch_exec) - invalid_design_id = ... + invalid_design_id = "INVALID" with pytest.raises(slurm.JobError) as exc_info: slurm.sbatch_map( From 2f607a97a903c2104240fccb1daea9c54c190bad Mon Sep 17 00:00:00 2001 From: Florian Felten Date: Tue, 14 Oct 2025 10:38:34 +0200 Subject: [PATCH 5/6] Avoid pickling traceback --- engibench/utils/slurm/run_job.py | 1 + tests/utils/test_slurm.py | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/engibench/utils/slurm/run_job.py b/engibench/utils/slurm/run_job.py index 6d9b00e6..03d0c483 100644 --- a/engibench/utils/slurm/run_job.py +++ b/engibench/utils/slurm/run_job.py @@ -49,6 +49,7 @@ def map_callback(**_kwargs) -> None: pickle.dump(MemorizeModule(result), out_stream) except Exception as e: # noqa: BLE001 with open(result_path, "wb") as out_stream: + e.__traceback__ = None # Python 3.13 now explicitly fails when trying to pickle the traceback pickle.dump(JobError(e, "Run job array item", args), out_stream) diff --git a/tests/utils/test_slurm.py b/tests/utils/test_slurm.py index 53290e99..3ee24bc5 100644 --- a/tests/utils/test_slurm.py +++ b/tests/utils/test_slurm.py @@ -165,7 +165,7 @@ def test_sbatch_map_reduce_with_exception(sbatch_exec: str) -> None: """Test if a fake slurm can process FakeProblem.""" slurm_args = slurm.SlurmConfig(sbatch_executable=sbatch_exec) - invalid_design_id = "INVALID" + invalid_design_id = ... with pytest.raises(slurm.JobError) as exc_info: slurm.sbatch_map( From e0787c64d94823bc3afdfd8ba0611e7a5e4f3181 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Gerhard=20Br=C3=A4unlich?= Date: Wed, 15 Oct 2025 10:51:55 +0200 Subject: [PATCH 6/6] utils.slurm: Make JobError objects pickleable with python 3.13 --- engibench/utils/slurm/__init__.py | 21 ++++++++++++++++++++- engibench/utils/slurm/run_job.py | 22 ++++++++++------------ 2 files changed, 30 insertions(+), 13 deletions(-) diff --git a/engibench/utils/slurm/__init__.py b/engibench/utils/slurm/__init__.py index 1e527540..234004d4 100644 --- a/engibench/utils/slurm/__init__.py +++ b/engibench/utils/slurm/__init__.py @@ -12,6 +12,7 @@ import subprocess import sys import tempfile +from traceback import StackSummary from traceback import TracebackException from typing import Any, Generic, TypeVar @@ -24,7 +25,7 @@ class JobError(Exception): - :attr:`origin` - Original exception instance. - :attr:`context` - Info (string) about which step failed (e.g. map, reduce or save). - :attr:`job_args` - dict containing the arguments passed to the job callback if the exception occurred during a job. - - :attr:`traceback` - `TracebackException `__ object. + - :attr:`traceback` - `TracebackException `_ object. """ def __init__(self, origin: Exception, context: str, job_args: dict[str, Any]) -> None: @@ -42,6 +43,24 @@ def __str__(self) -> str: """ +def dump_with_job_error(obj: Any, path: str) -> None: + """Pickle objects to a file which might contain a :py:class:`JobError` instance.""" + with open(path, "wb") as stream: + pickler = TracebackPickler(stream) + pickler.dump(obj) + + +class TracebackPickler(pickle.Pickler): + """Custom pickler to avoid pickling code objects when pickling tracebacks.""" + + def reducer_override(self, obj): + """Custom reducer for StackSummary.""" + if isinstance(obj, StackSummary): + return StackSummary.from_list, ([(s.filename, s.lineno, s.name, s.line) for s in obj],) + # For any other object, fallback to usual reduction + return NotImplemented + + if sys.version_info < (3, 11): class ExceptionGroup(Exception): # noqa: N818 diff --git a/engibench/utils/slurm/run_job.py b/engibench/utils/slurm/run_job.py index 03d0c483..301432fe 100644 --- a/engibench/utils/slurm/run_job.py +++ b/engibench/utils/slurm/run_job.py @@ -7,6 +7,7 @@ import sys from typing import Any +from engibench.utils.slurm import dump_with_job_error from engibench.utils.slurm import JobError from engibench.utils.slurm import MemorizeModule @@ -40,33 +41,31 @@ def map_callback(**_kwargs) -> None: with open(os.path.join(work_dir, "jobs", f"{index}.pkl"), "rb") as stream: args = pickle.load(stream) except Exception as e: # noqa: BLE001 - with open(result_path, "wb") as out_stream: - pickle.dump(JobError(e, "Unpickle job array item", {}), out_stream) + dump_with_job_error(JobError(e, "Unpickle job array item", {}), result_path) continue try: result = map_callback(**args) - with open(result_path, "wb") as out_stream: - pickle.dump(MemorizeModule(result), out_stream) + dump_with_job_error(MemorizeModule(result), result_path) except Exception as e: # noqa: BLE001 - with open(result_path, "wb") as out_stream: - e.__traceback__ = None # Python 3.13 now explicitly fails when trying to pickle the traceback - pickle.dump(JobError(e, "Run job array item", args), out_stream) + dump_with_job_error(JobError(e, "Run job array item", args), result_path) def reduce_job_results(work_dir: str, n_jobs: int) -> None: """Collect all results or errors from job array jobs, passing to a reduce callback.""" results = [] # prepare empty list for error, occurring before `results` is assigned a value + reduced_pkl = os.path.join(work_dir, "reduced.pkl") try: with open(os.path.join(work_dir, "jobs", "reduce.pkl"), "rb") as in_stream: reduce_callback = pickle.load(in_stream) results = collect_jobs(work_dir, n_jobs) reduced = reduce_callback(results) + dump_with_job_error(MemorizeModule(reduced), reduced_pkl) except Exception as e: # noqa: BLE001 errors = [e] + [err for err in results if isinstance(err, Exception)] - reduced = JobError(ExceptionGroup("", errors), "reduce", {}) if errors else JobError(e, "reduce", {}) - with open(os.path.join(work_dir, "reduced.pkl"), "wb") as out_stream: - pickle.dump(MemorizeModule(reduced), out_stream) + dump_with_job_error( + JobError(ExceptionGroup("", errors), "reduce", {}) if errors else JobError(e, "reduce", {}), reduced_pkl + ) def save(work_dir: str, n_jobs: int, out: str) -> None: @@ -75,8 +74,7 @@ def save(work_dir: str, n_jobs: int, out: str) -> None: if not any(isinstance(r, JobError) for r in results): shutil.rmtree(work_dir) - with open(out, "wb") as out_stream: - pickle.dump(results, out_stream) + dump_with_job_error(results, out) def collect_jobs(work_dir: str, n_jobs: int) -> list[Any]: