From 83f8bc88911a6fd525c9e3627ebe67142240bf75 Mon Sep 17 00:00:00 2001 From: rvalerio Date: Tue, 21 Oct 2025 15:05:55 +0100 Subject: [PATCH 01/13] Add unified dockerfile --- Dockerfile | 26 ++++++++++++++++++++++++++ file-tracker/setup.py | 2 +- start_services.sh | 35 +++++++++++++++++++++++++++++++++++ 3 files changed, 62 insertions(+), 1 deletion(-) create mode 100644 Dockerfile create mode 100644 start_services.sh diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 00000000..4fb8a7c5 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,26 @@ +FROM inductiva/task-runner:latest + +USER root + +# Install file-tracker +COPY /file-tracker/requirements.txt /file-tracker-requirements.txt +RUN pip install --no-cache-dir --upgrade -r /file-tracker-requirements.txt + +COPY /file-tracker /file-tracker +WORKDIR /file-tracker +RUN pip install . + +ENV FILE_TRACKER_HOST=0.0.0.0 +ENV FILE_TRACKER_PORT=5000 + +EXPOSE 5000 + +# Create startup script +WORKDIR / +COPY /start_services.sh /start_services.sh +RUN chmod +x /start_services.sh +RUN chown task-runner:task-runner /start_services.sh + +USER task-runner + +CMD ["/start_services.sh"] diff --git a/file-tracker/setup.py b/file-tracker/setup.py index 7c32adff..6b12bc4b 100644 --- a/file-tracker/setup.py +++ b/file-tracker/setup.py @@ -2,7 +2,7 @@ from setuptools import find_packages, setup setup( - name="inductiva-api-task-runner", + name="inductiva-api-file-tracker", packages=find_packages(), version="0.1.0", ) diff --git a/start_services.sh b/start_services.sh new file mode 100644 index 00000000..0689047b --- /dev/null +++ b/start_services.sh @@ -0,0 +1,35 @@ +#!/bin/bash + +# Function to handle cleanup on exit +cleanup() { + echo "Shutting down services..." + jobs -p | xargs -r kill + exit 0 +} + +# Set up signal handlers +trap cleanup SIGTERM SIGINT + +echo "Starting combined task-runner and file-tracker services..." + +# Start file-tracker in the background +echo "Starting file-tracker..." +cd /file-tracker +python ./file_tracker/main.py & +FILE_TRACKER_PID=$! + +# Start task-runner in the background +echo "Starting task-runner..." +cd /task-runner +python ./task_runner/main.py & +TASK_RUNNER_PID=$! + +echo "Both services started:" +echo " - File-tracker PID: $FILE_TRACKER_PID" +echo " - Task-runner PID: $TASK_RUNNER_PID" + +# Wait for either process to exit +wait $FILE_TRACKER_PID $TASK_RUNNER_PID + +echo "One of the services exited. Cleaning up..." +cleanup From 2f57359162083079f805a56d3206c4ceedf00d4a Mon Sep 17 00:00:00 2001 From: rvalerio Date: Wed, 22 Oct 2025 10:01:44 +0100 Subject: [PATCH 02/13] Improve combined mode --- .github/workflows/build-push-docker-image.yml | 30 +++++++++++++++ Makefile | 13 +++++++ README.md | 7 ++++ Dockerfile => combined/Dockerfile | 0 combined/start_services.sh | 37 +++++++++++++++++++ docker-compose.combined.yml | 22 +++++++++++ 6 files changed, 109 insertions(+) rename Dockerfile => combined/Dockerfile (100%) create mode 100644 combined/start_services.sh create mode 100644 docker-compose.combined.yml diff --git a/.github/workflows/build-push-docker-image.yml b/.github/workflows/build-push-docker-image.yml index d7629191..fb285c86 100644 --- a/.github/workflows/build-push-docker-image.yml +++ b/.github/workflows/build-push-docker-image.yml @@ -9,6 +9,7 @@ on: paths: - 'task-runner/**' - 'file-tracker/**' + - 'combined/**' jobs: build-publish-docker-images: @@ -54,6 +55,21 @@ jobs: org.opencontainers.image.title=File Tracker org.opencontainers.image.description=File tracker for the Inductiva API + # Metadata for task-runner-combined + - name: Docker meta for task-runner-combined + id: meta-task-runner-combined + uses: docker/metadata-action@v5 + with: + images: inductiva/task-runner-combined + tags: | + type=ref,event=branch + type=raw,value=latest,enable=${{ github.ref == format('refs/heads/{0}', 'main') }} + labels: | + org.opencontainers.image.url=https://inductiva.ai/ + org.opencontainers.image.source=${{ github.repository }} + org.opencontainers.image.title=Task Runner Combined + org.opencontainers.image.description=Task runner with file-tracker combined for the Inductiva API + - name: Set up Docker Buildx uses: docker/setup-buildx-action@v3 @@ -85,3 +101,17 @@ jobs: build-args: | API_URL=${{ github.ref == 'refs/heads/main' && 'https://api.inductiva.ai' || 'https://api-dev.inductiva.ai' }} + # Build and push task-runner-combined + - name: Build and push task-runner-combined to Docker Hub + uses: docker/build-push-action@v5 + with: + context: . + file: ./combined/Dockerfile + platforms: linux/amd64 + push: true + tags: ${{ steps.meta-task-runner-combined.outputs.tags }} + labels: | + ${{ steps.meta-task-runner-combined.outputs.labels }} + build-args: | + API_URL=${{ github.ref == 'refs/heads/main' && 'https://api.inductiva.ai' || 'https://api-dev.inductiva.ai' }} + diff --git a/Makefile b/Makefile index 816d7d7b..6240d516 100644 --- a/Makefile +++ b/Makefile @@ -23,6 +23,11 @@ DOCKER_COMPOSE_COMMAND_TASK_RUNNER_LITE=\ -p task-runner-lite-$(UID) \ -f docker-compose.lite.yml +DOCKER_COMPOSE_COMMAND_TASK_RUNNER_COMBINED=\ + $(DOCKER_COMPOSE_COMMAND) \ + -p task-runner-combined-$(UID) \ + -f docker-compose.combined.yml + .PHONY: % %: help @@ -32,9 +37,11 @@ help: @echo " make task-runner-up: starts task-runner building from source" @echo " make task-runner-lite-up: starts task-runner in lite mode (faster)" @echo " make task-runner-cuda-up: starts task-runner with CUDA support" + @echo " make task-runner-combined-up: starts task-runner with file-tracker combined" @echo " make task-runner-down stops task-runner building from source" @echo " make task-runner-lite-down stops task-runner in lite mode" @echo " make task-runner-cuda-down stops task-runner with CUDA support" + @echo " make task-runner-combined-down stops task-runner with file-tracker combined" @echo Utils: @echo " make lint-fix: run linter and fix issues" @echo " make format: run formatter" @@ -53,6 +60,9 @@ task-runner-lite-up:setup-apptainer-folder task-runner-cuda-up:setup-apptainer-folder $(DOCKER_COMPOSE_COMMAND_TASK_RUNNER_CUDA) up --build +task-runner-combined-up:setup-apptainer-folder + $(DOCKER_COMPOSE_COMMAND_TASK_RUNNER_COMBINED) up --build + task-runner-down: $(DOCKER_COMPOSE_COMMAND_TASK_RUNNER) down @@ -62,6 +72,9 @@ task-runner-lite-down: task-runner-cuda-down: $(DOCKER_COMPOSE_COMMAND_TASK_RUNNER_CUDA) down +task-runner-combined-down: + $(DOCKER_COMPOSE_COMMAND_TASK_RUNNER_COMBINED) down + lint-fix: ruff check --config=./pyproject.toml --fix diff --git a/README.md b/README.md index af8ed664..cd806020 100644 --- a/README.md +++ b/README.md @@ -60,6 +60,13 @@ Build and run the Task Runner with CUDA support: make task-runner-cuda-up ``` +#### Combined mode +Build and run the Task Runner with file-tracker combined in a single container: + +``` +make task-runner-combined-up +``` + ### Run Simulations You can now run simulations locally by passing a local machine when you call the `run` function. Try out the following example: diff --git a/Dockerfile b/combined/Dockerfile similarity index 100% rename from Dockerfile rename to combined/Dockerfile diff --git a/combined/start_services.sh b/combined/start_services.sh new file mode 100644 index 00000000..7871a0f5 --- /dev/null +++ b/combined/start_services.sh @@ -0,0 +1,37 @@ +#!/bin/bash + +# Function to handle cleanup on exit +cleanup() { + echo "Shutting down services..." + # Kill background processes + jobs -p | xargs -r kill + exit 0 +} + +# Set up signal handlers +trap cleanup SIGTERM SIGINT + +echo "Starting combined task-runner and file-tracker services..." + +# Start file-tracker in the background +echo "Starting file-tracker..." +cd /file-tracker +python ./file_tracker/main.py & +FILE_TRACKER_PID=$! + +# Start task-runner in the background +echo "Starting task-runner..." +cd /task-runner +python ./task_runner/main.py & +TASK_RUNNER_PID=$! + +echo "Both services started:" +echo " - File-tracker PID: $FILE_TRACKER_PID" +echo " - Task-runner PID: $TASK_RUNNER_PID" + +# Wait for either process to exit +wait $FILE_TRACKER_PID $TASK_RUNNER_PID + +# If we get here, one of the services exited +echo "One of the services exited. Cleaning up..." +cleanup diff --git a/docker-compose.combined.yml b/docker-compose.combined.yml new file mode 100644 index 00000000..01475dc0 --- /dev/null +++ b/docker-compose.combined.yml @@ -0,0 +1,22 @@ +services: + task-runner-combined: + build: + context: . + dockerfile: combined/Dockerfile + environment: + EXECUTER_IMAGES_DIR: /executer-images + API_URL: ${INDUCTIVA_API_URL} + USER_API_KEY: ${INDUCTIVA_API_KEY} + MACHINE_GROUP_NAME: ${MACHINE_GROUP_NAME} + HOST_NAME: ${TASK_RUNNER_HOSTNAME:-${HOSTNAME}} + volumes: + - ./apptainer:/executer-images + - workdir:/workdir + network_mode: host + extra_hosts: + - "host.docker.internal:host-gateway" + privileged: true + platform: linux/amd64 + +volumes: + workdir: From 0b4947cf213e60f3d41b6d118c2bad5d9c35f53b Mon Sep 17 00:00:00 2001 From: rvalerio Date: Wed, 22 Oct 2025 11:05:42 +0100 Subject: [PATCH 03/13] Rename combined to unified --- .github/workflows/build-push-docker-image.yml | 24 ++++++------- Makefile | 18 +++++----- README.md | 6 ++-- ...combined.yml => docker-compose.unified.yml | 4 +-- start_services.sh | 35 ------------------- {combined => unified}/Dockerfile | 2 +- {combined => unified}/start_services.sh | 2 +- 7 files changed, 28 insertions(+), 63 deletions(-) rename docker-compose.combined.yml => docker-compose.unified.yml (89%) delete mode 100644 start_services.sh rename {combined => unified}/Dockerfile (91%) rename {combined => unified}/start_services.sh (92%) diff --git a/.github/workflows/build-push-docker-image.yml b/.github/workflows/build-push-docker-image.yml index fb285c86..5500fe2e 100644 --- a/.github/workflows/build-push-docker-image.yml +++ b/.github/workflows/build-push-docker-image.yml @@ -9,7 +9,7 @@ on: paths: - 'task-runner/**' - 'file-tracker/**' - - 'combined/**' + - 'unified/**' jobs: build-publish-docker-images: @@ -55,20 +55,20 @@ jobs: org.opencontainers.image.title=File Tracker org.opencontainers.image.description=File tracker for the Inductiva API - # Metadata for task-runner-combined - - name: Docker meta for task-runner-combined - id: meta-task-runner-combined + # Metadata for task-runner-unified + - name: Docker meta for task-runner-unified + id: meta-task-runner-unified uses: docker/metadata-action@v5 with: - images: inductiva/task-runner-combined + images: inductiva/task-runner-unified tags: | type=ref,event=branch type=raw,value=latest,enable=${{ github.ref == format('refs/heads/{0}', 'main') }} labels: | org.opencontainers.image.url=https://inductiva.ai/ org.opencontainers.image.source=${{ github.repository }} - org.opencontainers.image.title=Task Runner Combined - org.opencontainers.image.description=Task runner with file-tracker combined for the Inductiva API + org.opencontainers.image.title=Task Runner Unified + org.opencontainers.image.description=Task runner with file-tracker unified for the Inductiva API - name: Set up Docker Buildx uses: docker/setup-buildx-action@v3 @@ -101,17 +101,17 @@ jobs: build-args: | API_URL=${{ github.ref == 'refs/heads/main' && 'https://api.inductiva.ai' || 'https://api-dev.inductiva.ai' }} - # Build and push task-runner-combined - - name: Build and push task-runner-combined to Docker Hub + # Build and push task-runner-unified + - name: Build and push task-runner-unified to Docker Hub uses: docker/build-push-action@v5 with: context: . - file: ./combined/Dockerfile + file: ./unified/Dockerfile platforms: linux/amd64 push: true - tags: ${{ steps.meta-task-runner-combined.outputs.tags }} + tags: ${{ steps.meta-task-runner-unified.outputs.tags }} labels: | - ${{ steps.meta-task-runner-combined.outputs.labels }} + ${{ steps.meta-task-runner-unified.outputs.labels }} build-args: | API_URL=${{ github.ref == 'refs/heads/main' && 'https://api.inductiva.ai' || 'https://api-dev.inductiva.ai' }} diff --git a/Makefile b/Makefile index 6240d516..29d587ec 100644 --- a/Makefile +++ b/Makefile @@ -23,10 +23,10 @@ DOCKER_COMPOSE_COMMAND_TASK_RUNNER_LITE=\ -p task-runner-lite-$(UID) \ -f docker-compose.lite.yml -DOCKER_COMPOSE_COMMAND_TASK_RUNNER_COMBINED=\ +DOCKER_COMPOSE_COMMAND_TASK_RUNNER_UNIFIED=\ $(DOCKER_COMPOSE_COMMAND) \ - -p task-runner-combined-$(UID) \ - -f docker-compose.combined.yml + -p task-runner-unified-$(UID) \ + -f docker-compose.unified.yml .PHONY: % @@ -37,11 +37,11 @@ help: @echo " make task-runner-up: starts task-runner building from source" @echo " make task-runner-lite-up: starts task-runner in lite mode (faster)" @echo " make task-runner-cuda-up: starts task-runner with CUDA support" - @echo " make task-runner-combined-up: starts task-runner with file-tracker combined" + @echo " make task-runner-unified-up: starts task-runner with file-tracker unified" @echo " make task-runner-down stops task-runner building from source" @echo " make task-runner-lite-down stops task-runner in lite mode" @echo " make task-runner-cuda-down stops task-runner with CUDA support" - @echo " make task-runner-combined-down stops task-runner with file-tracker combined" + @echo " make task-runner-unified-down stops task-runner with file-tracker unified" @echo Utils: @echo " make lint-fix: run linter and fix issues" @echo " make format: run formatter" @@ -60,8 +60,8 @@ task-runner-lite-up:setup-apptainer-folder task-runner-cuda-up:setup-apptainer-folder $(DOCKER_COMPOSE_COMMAND_TASK_RUNNER_CUDA) up --build -task-runner-combined-up:setup-apptainer-folder - $(DOCKER_COMPOSE_COMMAND_TASK_RUNNER_COMBINED) up --build +task-runner-unified-up:setup-apptainer-folder + $(DOCKER_COMPOSE_COMMAND_TASK_RUNNER_UNIFIED) up --build task-runner-down: $(DOCKER_COMPOSE_COMMAND_TASK_RUNNER) down @@ -72,8 +72,8 @@ task-runner-lite-down: task-runner-cuda-down: $(DOCKER_COMPOSE_COMMAND_TASK_RUNNER_CUDA) down -task-runner-combined-down: - $(DOCKER_COMPOSE_COMMAND_TASK_RUNNER_COMBINED) down +task-runner-unified-down: + $(DOCKER_COMPOSE_COMMAND_TASK_RUNNER_UNIFIED) down lint-fix: ruff check --config=./pyproject.toml --fix diff --git a/README.md b/README.md index cd806020..61778f3b 100644 --- a/README.md +++ b/README.md @@ -60,11 +60,11 @@ Build and run the Task Runner with CUDA support: make task-runner-cuda-up ``` -#### Combined mode -Build and run the Task Runner with file-tracker combined in a single container: +#### Unified mode +Build and run the Task Runner with file-tracker unified in a single container: ``` -make task-runner-combined-up +make task-runner-unified-up ``` ### Run Simulations diff --git a/docker-compose.combined.yml b/docker-compose.unified.yml similarity index 89% rename from docker-compose.combined.yml rename to docker-compose.unified.yml index 01475dc0..8b0f4bdc 100644 --- a/docker-compose.combined.yml +++ b/docker-compose.unified.yml @@ -1,8 +1,8 @@ services: - task-runner-combined: + task-runner-unified: build: context: . - dockerfile: combined/Dockerfile + dockerfile: unified/Dockerfile environment: EXECUTER_IMAGES_DIR: /executer-images API_URL: ${INDUCTIVA_API_URL} diff --git a/start_services.sh b/start_services.sh deleted file mode 100644 index 0689047b..00000000 --- a/start_services.sh +++ /dev/null @@ -1,35 +0,0 @@ -#!/bin/bash - -# Function to handle cleanup on exit -cleanup() { - echo "Shutting down services..." - jobs -p | xargs -r kill - exit 0 -} - -# Set up signal handlers -trap cleanup SIGTERM SIGINT - -echo "Starting combined task-runner and file-tracker services..." - -# Start file-tracker in the background -echo "Starting file-tracker..." -cd /file-tracker -python ./file_tracker/main.py & -FILE_TRACKER_PID=$! - -# Start task-runner in the background -echo "Starting task-runner..." -cd /task-runner -python ./task_runner/main.py & -TASK_RUNNER_PID=$! - -echo "Both services started:" -echo " - File-tracker PID: $FILE_TRACKER_PID" -echo " - Task-runner PID: $TASK_RUNNER_PID" - -# Wait for either process to exit -wait $FILE_TRACKER_PID $TASK_RUNNER_PID - -echo "One of the services exited. Cleaning up..." -cleanup diff --git a/combined/Dockerfile b/unified/Dockerfile similarity index 91% rename from combined/Dockerfile rename to unified/Dockerfile index 4fb8a7c5..795b6c8f 100644 --- a/combined/Dockerfile +++ b/unified/Dockerfile @@ -17,7 +17,7 @@ EXPOSE 5000 # Create startup script WORKDIR / -COPY /start_services.sh /start_services.sh +COPY /unified/start_services.sh /start_services.sh RUN chmod +x /start_services.sh RUN chown task-runner:task-runner /start_services.sh diff --git a/combined/start_services.sh b/unified/start_services.sh similarity index 92% rename from combined/start_services.sh rename to unified/start_services.sh index 7871a0f5..863338bf 100644 --- a/combined/start_services.sh +++ b/unified/start_services.sh @@ -11,7 +11,7 @@ cleanup() { # Set up signal handlers trap cleanup SIGTERM SIGINT -echo "Starting combined task-runner and file-tracker services..." +echo "Starting unified task-runner and file-tracker services..." # Start file-tracker in the background echo "Starting file-tracker..." From fe3da04c5a26c9d79886717d84cf88b9388a147b Mon Sep 17 00:00:00 2001 From: Bruno Grego Date: Wed, 22 Oct 2025 15:53:46 +0100 Subject: [PATCH 04/13] Fix build-publish-docker-images --- .github/workflows/build-push-docker-image.yml | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/.github/workflows/build-push-docker-image.yml b/.github/workflows/build-push-docker-image.yml index 5500fe2e..c020a379 100644 --- a/.github/workflows/build-push-docker-image.yml +++ b/.github/workflows/build-push-docker-image.yml @@ -32,8 +32,7 @@ jobs: with: images: inductiva/task-runner tags: | - type=ref,event=branch - type=raw,value=latest,enable=${{ github.ref == format('refs/heads/{0}', 'main') }} + type=raw,value=unified_${{ github.ref_name }} labels: | org.opencontainers.image.url=https://inductiva.ai/ org.opencontainers.image.source=${{ github.repository }} @@ -60,7 +59,7 @@ jobs: id: meta-task-runner-unified uses: docker/metadata-action@v5 with: - images: inductiva/task-runner-unified + images: inductiva/task-runner tags: | type=ref,event=branch type=raw,value=latest,enable=${{ github.ref == format('refs/heads/{0}', 'main') }} From c435c0f2bbf474aed069178e76b992380cd4eb40 Mon Sep 17 00:00:00 2001 From: Bruno Grego Date: Wed, 22 Oct 2025 15:57:37 +0100 Subject: [PATCH 05/13] Add workflow dispatch --- .github/workflows/build-push-docker-image.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/build-push-docker-image.yml b/.github/workflows/build-push-docker-image.yml index c020a379..bf191da5 100644 --- a/.github/workflows/build-push-docker-image.yml +++ b/.github/workflows/build-push-docker-image.yml @@ -1,6 +1,8 @@ name: Build Docker images and push to Docker Hub on: + workflow_dispatch: + push: branches: - main From 7f168b017bda77c068cc5dc6bca153f57aa41c55 Mon Sep 17 00:00:00 2001 From: Bruno Grego Date: Wed, 22 Oct 2025 15:59:48 +0100 Subject: [PATCH 06/13] Fix label --- .github/workflows/build-push-docker-image.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/build-push-docker-image.yml b/.github/workflows/build-push-docker-image.yml index bf191da5..eec5676f 100644 --- a/.github/workflows/build-push-docker-image.yml +++ b/.github/workflows/build-push-docker-image.yml @@ -34,7 +34,8 @@ jobs: with: images: inductiva/task-runner tags: | - type=raw,value=unified_${{ github.ref_name }} + type=ref,event=branch + type=raw,value=latest,enable=${{ github.ref == format('refs/heads/{0}', 'main') }} labels: | org.opencontainers.image.url=https://inductiva.ai/ org.opencontainers.image.source=${{ github.repository }} @@ -63,8 +64,7 @@ jobs: with: images: inductiva/task-runner tags: | - type=ref,event=branch - type=raw,value=latest,enable=${{ github.ref == format('refs/heads/{0}', 'main') }} + type=raw,value=unified_${{ github.ref_name }} labels: | org.opencontainers.image.url=https://inductiva.ai/ org.opencontainers.image.source=${{ github.repository }} From c9194f59a05289e4d2b4f311c2e64f5ed484ea24 Mon Sep 17 00:00:00 2001 From: Henrique Preto Date: Wed, 5 Nov 2025 10:38:19 +0000 Subject: [PATCH 07/13] Replace copy with move --- .../task_runner/executers/arbitrary_commands_executer.py | 2 +- task-runner/task_runner/executers/mpi_base_executer.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/task-runner/task_runner/executers/arbitrary_commands_executer.py b/task-runner/task_runner/executers/arbitrary_commands_executer.py index 6b75382c..be1136ec 100644 --- a/task-runner/task_runner/executers/arbitrary_commands_executer.py +++ b/task-runner/task_runner/executers/arbitrary_commands_executer.py @@ -28,7 +28,7 @@ def execute(self): self.args.run_subprocess_dir) # Copy the input files to the artifacts directory - shutil.copytree(input_dir, self.artifacts_dir, dirs_exist_ok=True) + shutil.move(src=input_dir, dst=self.artifacts_dir) original_username = None if self.commands_user: diff --git a/task-runner/task_runner/executers/mpi_base_executer.py b/task-runner/task_runner/executers/mpi_base_executer.py index 71e9bcdc..457ce50a 100644 --- a/task-runner/task_runner/executers/mpi_base_executer.py +++ b/task-runner/task_runner/executers/mpi_base_executer.py @@ -63,4 +63,4 @@ def execute(self): is_mpi=True) self.run_subprocess(cmd, working_dir=sim_dir) - shutil.copytree(sim_dir, self.artifacts_dir, dirs_exist_ok=True) + shutil.move(src=sim_dir, dst=self.artifacts_dir) From 18cc3f1ddde394a812654a5cb74bc830fe98233d Mon Sep 17 00:00:00 2001 From: Henrique Preto Date: Wed, 5 Nov 2025 15:37:12 +0000 Subject: [PATCH 08/13] Revert "Replace copy with move" This reverts commit c9194f59a05289e4d2b4f311c2e64f5ed484ea24. --- .../task_runner/executers/arbitrary_commands_executer.py | 2 +- task-runner/task_runner/executers/mpi_base_executer.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/task-runner/task_runner/executers/arbitrary_commands_executer.py b/task-runner/task_runner/executers/arbitrary_commands_executer.py index be1136ec..6b75382c 100644 --- a/task-runner/task_runner/executers/arbitrary_commands_executer.py +++ b/task-runner/task_runner/executers/arbitrary_commands_executer.py @@ -28,7 +28,7 @@ def execute(self): self.args.run_subprocess_dir) # Copy the input files to the artifacts directory - shutil.move(src=input_dir, dst=self.artifacts_dir) + shutil.copytree(input_dir, self.artifacts_dir, dirs_exist_ok=True) original_username = None if self.commands_user: diff --git a/task-runner/task_runner/executers/mpi_base_executer.py b/task-runner/task_runner/executers/mpi_base_executer.py index 457ce50a..71e9bcdc 100644 --- a/task-runner/task_runner/executers/mpi_base_executer.py +++ b/task-runner/task_runner/executers/mpi_base_executer.py @@ -63,4 +63,4 @@ def execute(self): is_mpi=True) self.run_subprocess(cmd, working_dir=sim_dir) - shutil.move(src=sim_dir, dst=self.artifacts_dir) + shutil.copytree(sim_dir, self.artifacts_dir, dirs_exist_ok=True) From 8f58069150ca58abea3c96e24d728a8b296ae54c Mon Sep 17 00:00:00 2001 From: Henrique Preto Date: Wed, 5 Nov 2025 16:34:07 +0000 Subject: [PATCH 09/13] Move sim_dir into artifacts_dir when creating artifacts --- .../task_runner/executers/arbitrary_commands_executer.py | 6 ------ task-runner/task_runner/executers/base_executer.py | 9 ++++++++- 2 files changed, 8 insertions(+), 7 deletions(-) diff --git a/task-runner/task_runner/executers/arbitrary_commands_executer.py b/task-runner/task_runner/executers/arbitrary_commands_executer.py index 6b75382c..661f1aa7 100644 --- a/task-runner/task_runner/executers/arbitrary_commands_executer.py +++ b/task-runner/task_runner/executers/arbitrary_commands_executer.py @@ -2,7 +2,6 @@ import getpass import os -import shutil from task_runner import executers from task_runner.utils import files @@ -18,8 +17,6 @@ def execute(self): else: global_env = {} - input_dir = os.path.join(self.working_dir, self.args.sim_dir) - run_subprocess_dir = self.artifacts_dir_container if hasattr(self.args, @@ -27,9 +24,6 @@ def execute(self): run_subprocess_dir = os.path.join(self.artifacts_dir_container, self.args.run_subprocess_dir) - # Copy the input files to the artifacts directory - shutil.copytree(input_dir, self.artifacts_dir, dirs_exist_ok=True) - original_username = None if self.commands_user: original_username = getpass.getuser() diff --git a/task-runner/task_runner/executers/base_executer.py b/task-runner/task_runner/executers/base_executer.py index cb639c24..901c48e6 100644 --- a/task-runner/task_runner/executers/base_executer.py +++ b/task-runner/task_runner/executers/base_executer.py @@ -4,6 +4,7 @@ its usage. """ import os +import shutil import threading import time from abc import ABC, abstractmethod @@ -91,8 +92,14 @@ def __init__( self.args = named_tuple_constructor(**extra_params) logging.info("Working directory: %s", self.working_dir) + + # Move the inputs from sim_dir/ to artifacts_dir/ so that artifacts_dir/ + # already contains the inputs, avoiding the need for copying + shutil.move( + src=f"{self.working_dir}/{self.args.sim_dir}", + dest=self.artifacts_dir, + ) - os.makedirs(self.artifacts_dir) logging.info("Created output directory: %s", self.output_dir) logging.info("Created artifacts directory: %s", self.artifacts_dir) From 30ed5d7a09bfa2ce6ddd66e6f105df0837791215 Mon Sep 17 00:00:00 2001 From: Henrique Preto Date: Wed, 5 Nov 2025 16:35:58 +0000 Subject: [PATCH 10/13] Fix param name --- task-runner/task_runner/executers/base_executer.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/task-runner/task_runner/executers/base_executer.py b/task-runner/task_runner/executers/base_executer.py index 901c48e6..0f1c3332 100644 --- a/task-runner/task_runner/executers/base_executer.py +++ b/task-runner/task_runner/executers/base_executer.py @@ -97,7 +97,7 @@ def __init__( # already contains the inputs, avoiding the need for copying shutil.move( src=f"{self.working_dir}/{self.args.sim_dir}", - dest=self.artifacts_dir, + dst=self.artifacts_dir, ) logging.info("Created output directory: %s", self.output_dir) From 90346e4ade89b04253c186627e47737457c3d6fd Mon Sep 17 00:00:00 2001 From: Henrique Preto Date: Wed, 5 Nov 2025 16:41:13 +0000 Subject: [PATCH 11/13] Run ruff and yapf --- task-runner/task_runner/executers/base_executer.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/task-runner/task_runner/executers/base_executer.py b/task-runner/task_runner/executers/base_executer.py index 0f1c3332..873a9405 100644 --- a/task-runner/task_runner/executers/base_executer.py +++ b/task-runner/task_runner/executers/base_executer.py @@ -92,7 +92,7 @@ def __init__( self.args = named_tuple_constructor(**extra_params) logging.info("Working directory: %s", self.working_dir) - + # Move the inputs from sim_dir/ to artifacts_dir/ so that artifacts_dir/ # already contains the inputs, avoiding the need for copying shutil.move( From f5d95efcafffe03a7dc95b4592e7b59d892b9e10 Mon Sep 17 00:00:00 2001 From: Henrique Preto Date: Wed, 5 Nov 2025 16:44:53 +0000 Subject: [PATCH 12/13] Delete deprecated mpi executer --- .../executers/mpi_base_executer.py | 66 ------------------- 1 file changed, 66 deletions(-) delete mode 100644 task-runner/task_runner/executers/mpi_base_executer.py diff --git a/task-runner/task_runner/executers/mpi_base_executer.py b/task-runner/task_runner/executers/mpi_base_executer.py deleted file mode 100644 index 71e9bcdc..00000000 --- a/task-runner/task_runner/executers/mpi_base_executer.py +++ /dev/null @@ -1,66 +0,0 @@ -"""This file provides a class to implement MPI executers.""" -import os -import shutil -from typing import Any - -from task_runner import executers -from task_runner.executers import mpi_configuration - -# Instructions inside Docker containers are run by the root user (as default), -# so we need to allow Open MPI to be run as root. This is usually strongly -# discouraged, but necessary to run Open MPI inside a container. For further -# details, see https://www.open-mpi.org/doc/v4.1/man1/mpirun.1.php#toc25. -MPI_ALLOW = "--allow-run-as-root" -MPI_DISTRIBUTION_FILENAME = "machinefile" - - -class MPIExecuter(executers.BaseExecuter): - """Implementation of a general MPI Executer.""" - - def __init__( - self, - working_dir, - container_image, - mpi_config: mpi_configuration.MPIClusterConfiguration, - exec_command_logger: executers.ExecCommandLogger, - extra_params: dict[str, Any], - sim_binary, - file_type, - sim_specific_input_filename, - ): - super().__init__(working_dir, container_image, mpi_config, - exec_command_logger, extra_params) - self.sim_binary = sim_binary - self.sim_specific_input_filename = sim_specific_input_filename - self.file_type = file_type - - def execute(self): - sim_dir = os.path.join(self.working_dir, self.args.sim_dir) - input_filename = self.args.input_filename - - input_file_full_path = os.path.join(sim_dir, input_filename) - - if not os.path.exists(input_file_full_path): - if os.path.exists(f"{input_file_full_path}.{self.file_type}"): - input_filename = f"{input_file_full_path}.{self.file_type}" - else: - raise ValueError( - f"A file with name {input_filename} doesn't exist.") - - if self.args.n_vcpus: - self.mpi_config.extra_args.extend(["-np", f"{self.args.n_vcpus}"]) - - use_hwthread = bool(self.args.use_hwthread) - - if use_hwthread: - self.mpi_config.extra_args.extend(["--use-hwthread-cpus"]) - # Renaming input file as the simulator expects it to be - os.rename(input_file_full_path, - os.path.join(sim_dir, self.sim_specific_input_filename)) - - cmd = executers.Command(self.sim_binary + " " + - self.sim_specific_input_filename, - is_mpi=True) - self.run_subprocess(cmd, working_dir=sim_dir) - - shutil.copytree(sim_dir, self.artifacts_dir, dirs_exist_ok=True) From c5d9be22ea8a2822404b0751a94aa67c87f77db2 Mon Sep 17 00:00:00 2001 From: Henrique Preto Date: Wed, 5 Nov 2025 16:45:39 +0000 Subject: [PATCH 13/13] Delete mpi executer from init --- task-runner/task_runner/executers/__init__.py | 1 - 1 file changed, 1 deletion(-) diff --git a/task-runner/task_runner/executers/__init__.py b/task-runner/task_runner/executers/__init__.py index 18483513..f7e4facf 100644 --- a/task-runner/task_runner/executers/__init__.py +++ b/task-runner/task_runner/executers/__init__.py @@ -7,7 +7,6 @@ from .exec_command_logger import ExecCommandLogger # noqa: I001 from .base_executer import BaseExecuter, ExecuterSubProcessError # noqa: I001 from .command import Command # noqa: I001 -from .mpi_base_executer import MPIExecuter # noqa: I001 from .mpi_configuration import MPIClusterConfiguration # noqa: I001 from .subprocess_tracker import SubprocessTracker # noqa: I001 from . import (