Rebase "datasets" branch

.circleci/config.yml

@@ -38,6 +38,9 @@ jobs:
             docker compose -f docker-compose.yml -f docker-compose.selenium.yml exec django python manage.py collectstatic --noinput
 
       - run: docker-compose exec django flake8 src/
+      - run: docker pull codalab/codalab-legacy:py37 # not available without "not e2e" tests as they pull ahead of time
+      - run: docker pull codalab/codalab-legacy:py3 # not available without "not e2e" tests as they pull ahead of time
+      - run: docker pull vergilgxw/autotable:v2 # not available without "not e2e" tests as they pull ahead of time
 
       - run:
           name: pytest
@@ -48,6 +51,13 @@ jobs:
           command: docker compose -f docker-compose.yml -f docker-compose.selenium.yml exec django py.test src/tests/functional/ -m e2e
           no_output_timeout: 60m
 
+      # Example to run specific set of tests (for debugging individual tests from a batch of tests)
+      # - run:
+      #     name: e2e tests - competitions
+      #     command: docker compose -f docker-compose.yml -f docker-compose.selenium.yml exec django py.test src/tests/functional/test_competitions.py -m e2e
+      #     no_output_timeout: 60m
+
+
       - store_artifacts:
           path: artifacts/
 

.github/workflows/ghPages-dev.yml

@@ -0,0 +1,25 @@
+name: github-pages-dev 
+on:
+  push:
+    branches:
+      - develop
+permissions:
+  contents: write
+jobs:
+  deploy-dev:
+    runs-on: ubuntu-latest
+    defaults:
+        run:
+            working-directory: documentation/
+    steps:
+      - uses: actions/checkout@v4
+      - name: Configure Git Credentials
+        run: |
+          git config user.name github-actions[bot]
+          git config user.email 41898282+github-actions[bot]@users.noreply.github.com
+      - uses: actions/setup-python@v5
+        with:
+          python-version: 3.x
+      - run: curl -LsSf https://astral.sh/uv/install.sh | sh
+      - run: /home/runner/.local/bin/uv sync 
+      - run: git fetch origin gh-pages --depth=1 && PDF=1 /home/runner/.local/bin/uv run mike deploy -u dev --push

.github/workflows/ghPages-prod.yml

@@ -0,0 +1,25 @@
+name: github-pages-prod
+on:
+  push:
+    tags:
+      - '*'
+permissions:
+  contents: write
+jobs:
+  deploy-prod:
+    runs-on: ubuntu-latest
+    defaults:
+        run:
+            working-directory: documentation/
+    steps:
+      - uses: actions/checkout@v4
+      - name: Configure Git Credentials
+        run: |
+          git config user.name github-actions[bot]
+          git config user.email 41898282+github-actions[bot]@users.noreply.github.com
+      - uses: actions/setup-python@v5
+        with:
+          python-version: 3.x
+      - run: curl -LsSf https://astral.sh/uv/install.sh | sh
+      - run: /home/runner/.local/bin/uv sync 
+      - run: git fetch origin gh-pages --depth=1 && PDF=1 /home/runner/.local/bin/uv run mike deploy -u  ${{ github.ref_name }} latest --push

Dockerfile.compute_worker

@@ -1,21 +1,28 @@
-FROM --platform=linux/amd64 python:3.9
+FROM --platform=linux/amd64 fedora:42
 
 # This makes output not buffer and return immediately, nice for seeing results in stdout
 ENV PYTHONUNBUFFERED 1
 
 # Install Docker
-RUN apt-get update && curl -fsSL https://get.docker.com | sh
+RUN dnf -y install dnf-plugins-core && \
+    dnf-3 config-manager --add-repo https://download.docker.com/linux/fedora/docker-ce.repo && \
+    dnf -y update && \
+    dnf install -y docker-ce docker-ce-cli containerd.io docker-buildx-plugin docker-compose-plugin && \
+    dnf install -y python3.9 && \
+    dnf clean all && \
+    rm -rf /var/cache /var/log/dnf* /var/log/yum.*
 
 
-RUN curl -sSL https://install.python-poetry.org | python3 - --version 1.8.3
+RUN curl -sSL https://install.python-poetry.org | python3.9 - --version 1.8.3
 # Poetry location so future commands (below) work
 ENV PATH $PATH:/root/.local/bin
 # Want poetry to use system python of docker container
 RUN poetry config virtualenvs.create false
 RUN poetry config virtualenvs.in-project false
 COPY ./compute_worker/pyproject.toml ./
 COPY ./compute_worker/poetry.lock ./
-RUN poetry install
+# To use python3.9 instead of system python
+RUN poetry config virtualenvs.prefer-active-python true && poetry install
 
 ADD compute_worker .
 

Dockerfile.compute_worker_gpu

@@ -1,38 +1,12 @@
-FROM --platform=linux/amd64 python:3.9
-
-# This makes output not buffer and return immediately, nice for seeing results in stdout
-ENV PYTHONUNBUFFERED 1
-
-# Install Docker
-RUN apt-get update && curl -fsSL https://get.docker.com | sh
-
-
+FROM --platform=linux/amd64 codalab/competitions-v2-compute-worker:latest
 # Nvidia Container Toolkit for cuda use with docker
 # [source](https://docs.nvidia.com/datacenter/cloud-native/container-toolkit/latest/install-guide.html)
-RUN curl -fsSL https://nvidia.github.io/libnvidia-container/gpgkey | gpg --dearmor -o /usr/share/keyrings/nvidia-container-toolkit-keyring.gpg \
-    && curl -s -L https://nvidia.github.io/libnvidia-container/stable/deb/nvidia-container-toolkit.list | \
-    sed 's#deb https://#deb [signed-by=/usr/share/keyrings/nvidia-container-toolkit-keyring.gpg] https://#g' | \
-    tee /etc/apt/sources.list.d/nvidia-container-toolkit.list
-RUN apt-get update -y;
-RUN apt-get install -y nvidia-container-toolkit
+# Include deps
+RUN dnf -y config-manager addrepo --from-repofile=https://nvidia.github.io/libnvidia-container/stable/rpm/nvidia-container-toolkit.repo && \
+    dnf -y update && \
+    dnf -y install nvidia-container-runtime nvidia-container-toolkit --exclude container-selinux && \
+    dnf clean all && \
+    rm -rf /var/cache /var/log/dnf* /var/log/yum.*
 # Make it explicit that we're using GPUs
 # BB - not convinced we need this
 ENV USE_GPU 1
-
-RUN curl -sSL https://install.python-poetry.org | python3 - --version 1.8.3
-# Poetry location so future commands (below) work
-ENV PATH $PATH:/root/.local/bin
-# Want poetry to use system python of docker container
-RUN poetry config virtualenvs.create false
-RUN poetry config virtualenvs.in-project false
-COPY ./compute_worker/pyproject.toml ./
-COPY ./compute_worker/poetry.lock ./
-RUN poetry install
-
-ADD compute_worker .
-
-CMD celery -A compute_worker worker \
-    -l info \
-    -Q compute-worker \
-    -n compute-worker@%n \
-    --concurrency=1

compute_worker/compute_worker.py

@@ -302,7 +302,9 @@ def get_detailed_results_file_path(self):
     async def send_detailed_results(self, file_path):
         logger.info(f"Updating detailed results {file_path} - {self.detailed_results_url}")
         self._put_file(self.detailed_results_url, file=file_path, content_type='text/html')
-        async with websockets.connect(self.websocket_url) as websocket:
+        websocket_url = f"{self.websocket_url}?kind=detailed_results"
+        logger.info(f"Connecting to {websocket_url} for detailed results")
+        async with websockets.connect(websocket_url) as websocket:
             await websocket.send(json.dumps({
                 "kind": 'detailed_result_update',
             }))
@@ -390,10 +392,14 @@ async def _send_data_through_socket(self, error_message):
         - Docker image pull failure logs
         - Execution time limit exceeded logs
         """
-        logger.info(f"Connecting to {self.websocket_url} to send docker image pull error")
+        # Create a unique websocket URL for error messages
+        websocket_url = f"{self.websocket_url}?kind=error_logs"
+        logger.info(f"Connecting to {websocket_url} to send error message")
+
+        logger.info(f"Connecting to {websocket_url} to send docker image pull error")
 
         # connect to web socket
-        websocket = await websockets.connect(self.websocket_url)
+        websocket = await websockets.connect(websocket_url)
 
         # define websocket errors
         websocket_errors = (socket.gaierror, websockets.WebSocketException, websockets.ConnectionClosedError, ConnectionRefusedError)
@@ -416,7 +422,7 @@ async def _send_data_through_socket(self, error_message):
             # no error in websocket message sending
             logger.info(f"Error sent successfully through websocket")
 
-        logger.info(f"Disconnecting from websocket {self.websocket_url}")
+        logger.info(f"Disconnecting from websocket {websocket_url}")
 
         # close websocket
         await websocket.close()
@@ -500,8 +506,11 @@ async def _run_container_engine_cmd(self, engine_cmd, kind):
         }
 
         # Start websocket, it will reconnect in the stdout/stderr listener loop below
-        logger.info(f"Connecting to {self.websocket_url}")
-        websocket = await websockets.connect(self.websocket_url)
+        # This ensures each task has its own independent WebSocket connection
+        websocket_url = f"{self.websocket_url}?kind={kind}"
+        logger.debug(f"WORKER_MARKER: Connecting to {websocket_url}")
+        websocket = await websockets.connect(websocket_url)
+        # websocket = await websockets.connect(self.websocket_url) # old BB
         websocket_errors = (socket.gaierror, websockets.WebSocketException, websockets.ConnectionClosedError, ConnectionRefusedError)
 
         # Function to read a line, if the line is larger than the buffer size we will
@@ -522,7 +531,7 @@ async def _readline_or_chunk(stream):
                 logs = [self.logs[kind][key] for key in ('stdout', 'stderr')]
                 for value in logs:
                     try:
-                        out = await asyncio.wait_for(_readline_or_chunk(value["stream"]), timeout=.1)
+                        out = await asyncio.wait_for(_readline_or_chunk(value["stream"]), timeout=0.1)
                         if out:
                             value["data"] += out
                             print("WS: " + str(out))
@@ -535,32 +544,36 @@ async def _readline_or_chunk(stream):
                     except asyncio.TimeoutError:
                         continue
             except websocket_errors:
+                logger.debug("\n\nWebsocket error (line 538)\n\n")
                 try:
                     # do we need to await websocket.close() on the old socket? before making a new one probably not?
                     await websocket.close()
                 except Exception as e:
                     logger.error(e)
-                    logger.info(e)
                     # TODO: catch proper exceptions here..! What can go wrong failing to close?
                     pass
 
                 # try to reconnect a few times
                 tries = 0
                 while tries < 3 and not websocket.open:
                     try:
-                        websocket = await websockets.connect(self.websocket_url)
+                        logger.debug(f"\n\nAttempting to reconnect in 2 seconds (attempt {tries+1}/3)")
+                        websocket = await websockets.connect(websocket_url)
+                        logger.debug(f"\n\nSuccessfully reconnected to {websocket_url}")
                     except websocket_errors:
+                        logger.error(f"\n\nReconnection attempt {tries+1} failed: {websocket_errors}")
                         await asyncio.sleep(2)
                         tries += 1
 
         self.logs[kind]["end"] = time.time()
 
-        logger.info(f"Process exited with {proc.returncode}")
-        logger.info(f"Disconnecting from websocket {self.websocket_url}")
+        logger.debug(f"Process exited with {proc.returncode}")
+        logger.debug(f"Disconnecting from websocket {websocket_url}")
 
         # Communicate that the program is closing
         self.completed_program_counter += 1
 
+        logger.debug(f"WORKER_MARKER: Disconnecting from {websocket_url}, program counter = {self.completed_program_counter}")
         await websocket.close()
 
     def _get_host_path(self, *paths):
@@ -609,6 +622,9 @@ async def _run_program_directory(self, program_dir, kind):
                 logger.info(
                     "Program directory missing metadata, assuming it's going to be handled by ingestion"
                 )
+                # Copy submission files into prediction output
+                # This is useful for results submissions but wrongly uses storage
+                shutil.copytree(program_dir, self.output_dir)
                 return
             else:
                 raise SubmissionException("Program directory missing 'metadata.yaml/metadata'")

docker-compose.selenium.yml

@@ -4,11 +4,12 @@ services:
     environment:
       - SELENIUM_HOSTNAME=selenium
       - SUBMISSIONS_API_URL=http://django:36475/api
+      - WEBSOCKET_ALLOWED_ORIGINS=*
     ports:
       - 36475:36475
 
   selenium:
-    image: selenium/standalone-firefox:124.0
+    image: selenium/standalone-firefox:120.0
     volumes:
       - ./src/tests/functional/test_files:/test_files/
       - ./artifacts:/artifacts/:z

docker-compose.yml

@@ -53,7 +53,7 @@ services:
   # Minio local storage helper
   #-----------------------------------------------
   minio:
-    image: minio/minio:RELEASE.2020-10-03T02-19-42Z
+    image: minio/minio:RELEASE.2025-04-22T22-12-26Z
     command: server /export
     volumes:
       - ./var/minio:/export
@@ -62,11 +62,11 @@ services:
       - $MINIO_PORT:9000
     env_file: .env
     healthcheck:
-      test: ["CMD", "nc", "-z", "minio", "9000"]
+      test: ["CMD", "curl", "-I", "http://minio:9000/minio/health/live"]
       interval: 5s
       retries: 5
   createbuckets:
-    image: minio/mc
+    image: minio/mc:RELEASE.2025-07-21T05-28-08Z
     depends_on:
       minio:
         condition: service_healthy
@@ -78,7 +78,7 @@ services:
       /bin/sh -c "
       set -x;
       if [ -n \"$MINIO_ACCESS_KEY\" ] && [ -n \"$MINIO_SECRET_KEY\" ] && [ -n \"$MINIO_PORT\" ]; then
-        until /usr/bin/mc config host add minio_docker http://minio:$MINIO_PORT $MINIO_ACCESS_KEY $MINIO_SECRET_KEY && break; do 
+        until /usr/bin/mc alias set minio_docker http://minio:$MINIO_PORT $MINIO_ACCESS_KEY $MINIO_SECRET_KEY && break; do 
           echo '...waiting...' && sleep 5; 
         done;
         /usr/bin/mc mb minio_docker/$AWS_STORAGE_BUCKET_NAME || echo 'Bucket $AWS_STORAGE_BUCKET_NAME already exists.';

documentation/.gitignore

@@ -0,0 +1,14 @@
+# Python-generated files
+__pycache__/
+*.py[oc]
+build/
+dist/
+wheels/
+*.egg-info
+.cache/
+# Virtual environments
+.venv
+
+uv.lock
+
+site/*

documentation/.python-version

@@ -0,0 +1 @@
+3.12

documentation/README.md

@@ -0,0 +1,39 @@
+## Codabench Documentation
+Welcome to the Codabench Documentation.
+
+You can access the documentation generated from this folder [here](https://codalab.org/codabench/latest/)
+
+If you want to contribute to the wiki, you can create a Pull Request modifying the files you want (located in `docs/`) while adding a quick explanation on what you have changed and why.
+
+
+When creating a Pull Request to modify the core code of Codabench, you can also include the wiki modification by modifying the relevant files. Once the Pull Request is merged, the wiki will automatically be updated with your changes (`dev` tag when it's merged into develop, and the `latest` version of the wiki once it's merged in master)
+
+## How to build
+To build the wiki locally, you will have to first install [uv](https://github.com/astral-sh/uv)
+
+Once that is done, you can run the following commands (while inside this folder):
+
+```bash
+uv sync # You only need to run this once, it will download all the necessary python packages
+PDF=1 uv run mkdocs serve -a localhost:8888 # This will build the site and serve it on localhost:8888
+```
+
+Open [localhost:8888](http://localhost:8888/) in your browser and you will see the wiki. Every changes you make will rebuild the documentation.
+You can remove the `PDF=1` environement variable if you want to speed up the build process, but you will not generate the related PDF.
+
+
+### Versioning
+We use the [mike](https://github.com/jimporter/mike) plugin to preserve multiple version of the wiki.
+
+To use it, you can run the following command: 
+```bash
+PDF=1 uv run mike deploy -u dev # This will create the site and push the changes in the gh-pages branch
+uv run mike serve -a localhost:8888 # Serve the site on localhost:8888
+```
+Check the official Github page of the plugin for more information on how it works.
+
+## Images and Assets
+Images and assets are saved in the `_attachments` folder closest to the documentation file that calls for the image. If an image is used in multiple different places, then it should be put in `_attachements` folder in the `docs/` root directory.
+
+## Github workflow
+We have Github workflows set up to automatically rebuild the wiki when the `develop` branch receives changes, and when a new tag is created for the `master` branch.

documentation/docs/Contribute/contributing.md

@@ -0,0 +1,21 @@
+## Being a Codabench user
+
+- Create a user account on [Codabench](https://codabench.org)
+- Register on [Codabench](https://codabench.org) to this existing competition [IRIS-tuto](https://www.codabench.org/competitions/1115/)  and make a submission (you can find the necessary files [here](https://github.com/codalab/competition-examples/tree/master/codabench/iris)): `sample_result_submission` and `sample_code_submission`. See [this page](../Participants/User_Participating-in-a-Competition.md) for more information.
+- Create your own private competition (you can find the necessary files [here](https://github.com/codalab/competition-examples/tree/master/codabench/) ). See [this page](../Organizers/Benchmark_Creation/Getting-started-with-Codabench.md) for more information.
+
+## Setting up a local instance of Codabench
+
+- Follow the tutorial in codabench [wiki](../Developers_and_Administrators/Codabench-Installation.md). According to your hosting OS, you might have to tune your environment file a bit. Try without enabling the SSL protocol (doing so, you don't need a domain name for the server). Try using the embedded Minio storage solution instead of a private cloud storage.
+- If needed, you can also look into [How to deploy Codabench on your server](../Developers_and_Administrators/How-to-deploy-Codabench-on-your-server.md)
+
+### Using your local instance
+
+- Create your own competition and play with it. You can look at the output logs of each different docker container.
+- Setting you as an [admin](../Developers_and_Administrators/Administrator-procedures.md#give-superuser-privileges-to-a-user) of your platform and visit the Django Admin menu.
+
+## Setting up an autonomous Compute Worker on a machine
+
+- Configure and launch a [compute worker](../Organizers/Running_a_benchmark/Compute-Worker-Management---Setup.md) docker container.
+- Create a private [Queue](../Organizers/Running_a_benchmark/Queue-Management.md#create-queue) on your new own competition on the production server codabench.org
+- Assign your own compute-worker to this private queue instead of the default queue.