[MNT] Dockerized tests for CI runs using localhost

.github/workflows/test.yml

@@ -101,6 +101,88 @@ jobs:
         echo "BEFORE=$git_status" >> $GITHUB_ENV
         echo "Repository status before tests: $git_status"
 
+    - name: Configure Host Network
+      if: matrix.os == 'ubuntu-latest'
+      run: |
+        # Map 'nginx' to localhost so the Host machine can resolve the URLs in the database
+        echo "127.0.0.1 nginx" | sudo tee -a /etc/hosts
+
+    - name: Clone Services & Apply Universal Patch
+      if: matrix.os == 'ubuntu-latest'
+      run: |
+        git clone --depth 1 https://github.com/openml/services.git
+        cd services
+        git config user.email "ci@openml.org"
+        git config user.name "CI"
+        git fetch origin pull/13/head:pr-13 && git merge pr-13 --no-edit
+        git fetch origin pull/15/head:pr-15 && git merge pr-15 --no-edit
+
+        # === PATCH 1: Use 'nginx' hostname ===
+        # This works inside Docker (DNS) and on Host (via /etc/hosts hack above)
+        sed -i 's/localhost:8000/nginx:8000/g' config/database/update.sh
+
+        # === PATCH 2: Fix Path Mismatch ===
+        # Ensure we use /data/ which Nginx recognizes
+        sed -i 's|/minio/|/data/|g' config/database/update.sh
+
+        echo "=== Patched Update Script ==="
+        cat config/database/update.sh | grep "nginx"
+
+    - name: Start Docker Services
+      if: matrix.os == 'ubuntu-latest'
+      working-directory: ./services
+      run: |
+        sudo systemctl stop mysql.service
+        docker compose --profile rest-api --profile minio --profile evaluation-engine up -d --build
+
+        echo "1. Waiting for Database population..."
+        docker wait openml-test-database-setup
+
+        echo "2. Waiting for Elasticsearch (this is the slow part)..."
+        # Wait up to 5 minutes for ES to go green
+        timeout 300s bash -c 'until [ "$(docker inspect -f {{.State.Health.Status}} openml-elasticsearch)" == "healthy" ]; do sleep 5; done'
+
+        echo "3. Waiting for PHP API..."
+        # Wait up to 5 minutes for PHP to accept connections
+        timeout 300s bash -c 'until [ "$(docker inspect -f {{.State.Health.Status}} openml-php-rest-api)" == "healthy" ]; do sleep 5; done'
+
+    - name: Finalize Setup & Verify Splits
+      if: matrix.os == 'ubuntu-latest'
+      run: |
+        echo "1. Forcing Elasticsearch Indexing Sync (With Auth)..."
+        # We append the default test API Key (AD0...0) to authorized the admin action
+        curl -s "http://nginx:8000/api/v1/xml/admin/index/sync?api_key=AD000000000000000000000000000000" || echo "Sync request failed"
+
+        echo "2. Waiting for Evaluation Engine to process Task 119..."
+        echo "Targeting Task 119 (The primary failure point)."
+
+        # We give it 5 minutes (300s) to handle the queue.
+        count=0
+        while [ $count -lt 30 ]; do
+           code=$(curl -s -o /dev/null -w "%{http_code}" http://nginx:8000/api_splits/get/119/Task_119_splits.arff)
+           if [ "$code" == "200" ]; then
+             echo "✅ SUCCESS: Task 119 splits are ready!"
+             exit 0
+           fi
+           echo "   ... waiting for split generation (Current Status: $code)"
+           sleep 10
+           count=$((count+1))
+        done
+
+        echo "❌ ERROR: Evaluation Engine timed out."
+
+        echo "=== DEBUG: LISTING LOG DIR ==="
+        docker exec openml-evaluation-engine ls -R /logs/ || echo "Dir empty"
+
+        echo "=== DEBUG: DUMPING ALL LOGS ==="
+        # Use wildcard to catch whatever the filename actually is
+        docker exec openml-evaluation-engine sh -c "cat /logs/*.log" || echo "Could not read logs"
+
+        # Also check the cron log again, just in case
+        docker exec openml-evaluation-engine cat /cron.log || echo "Cron log empty"
+
+        exit 1
+
     - name: Show installed dependencies
       run: python -m pip list
 
@@ -112,9 +194,9 @@ jobs:
         fi
 
         if [ "${{ matrix.sklearn-only }}" = "true" ]; then
-          marks="sklearn and not production and not uses_test_server"
+          marks="sklearn and not production"
         else
-          marks="not production and not uses_test_server"
+          marks="not production"
         fi
 
         pytest -n 4 --durations=20 --dist load -sv $codecov -o log_cli=true -m "$marks"
@@ -127,9 +209,9 @@ jobs:
         fi
 
         if [ "${{ matrix.sklearn-only }}" = "true" ]; then
-          marks="sklearn and production and not uses_test_server"
+          marks="sklearn and production"
         else
-          marks="production and not uses_test_server"
+          marks="production"
         fi
 
         pytest -n 4 --durations=20 --dist load -sv $codecov -o log_cli=true -m "$marks"
@@ -139,6 +221,20 @@ jobs:
       run: |  # we need a separate step because of the bash-specific if-statement in the previous one.
         pytest -n 4 --durations=20 --dist load -sv --reruns 5 --reruns-delay 1 -m "not uses_test_server"
 
+    - name: Upload coverage
+      if: matrix.code-cov && always()
+      uses: codecov/codecov-action@v4
+      with:
+        files: coverage.xml
+        token: ${{ secrets.CODECOV_TOKEN }}
+        fail_ci_if_error: true
+        verbose: true
+
+    - name: Cleanup Docker setup
+      if: matrix.os == 'ubuntu-latest' && always()
+      run: |
+        sudo rm -rf services
+
     - name: Check for files left behind by test
       if: matrix.os != 'windows-latest' && always()
       run: |
@@ -151,15 +247,6 @@ jobs:
             exit 1
         fi
 
-    - name: Upload coverage
-      if: matrix.code-cov && always()
-      uses: codecov/codecov-action@v4
-      with:
-        files: coverage.xml
-        token: ${{ secrets.CODECOV_TOKEN }}
-        fail_ci_if_error: true
-        verbose: true
-
   dummy_windows_py_sk024:
     name: (windows-latest, Py, sk0.24.*, sk-only:false)
     runs-on: ubuntu-latest

openml/cli.py

@@ -109,7 +109,7 @@ def check_server(server: str) -> str:
 
     def replace_shorthand(server: str) -> str:
         if server == "test":
-            return "https://test.openml.org/api/v1/xml"
+            return f"{config.TEST_SERVER_URL}/api/v1/xml"
         if server == "production":
             return "https://www.openml.org/api/v1/xml"
         return server

openml/config.py

@@ -9,6 +9,7 @@
 import os
 import platform
 import shutil
+import sys
 import warnings
 from collections.abc import Iterator
 from contextlib import contextmanager
@@ -27,6 +28,11 @@
 OPENML_SKIP_PARQUET_ENV_VAR = "OPENML_SKIP_PARQUET"
 _TEST_SERVER_NORMAL_USER_KEY = "normaluser"
 
+if sys.platform.startswith("win"):
+    TEST_SERVER_URL = "http://localhost"
+else:
+    TEST_SERVER_URL = "http://localhost:8000"
+
 
 class _Config(TypedDict):
     apikey: str
@@ -213,7 +219,7 @@ class ConfigurationForExamples:
     _last_used_server = None
     _last_used_key = None
     _start_last_called = False
-    _test_server = "https://test.openml.org/api/v1/xml"
+    _test_server = f"{TEST_SERVER_URL}/api/v1/xml"
     _test_apikey = _TEST_SERVER_NORMAL_USER_KEY
 
     @classmethod

openml/tasks/functions.py

@@ -415,9 +415,10 @@ def get_task(
     if not isinstance(task_id, int):
         raise TypeError(f"Task id should be integer, is {type(task_id)}")
 
-    cache_key_dir = openml.utils._create_cache_directory_for_id(TASKS_CACHE_DIR_NAME, task_id)
-    tid_cache_dir = cache_key_dir / str(task_id)
-    tid_cache_dir_existed = tid_cache_dir.exists()
+    task_cache_directory = openml.utils._create_cache_directory_for_id(
+        TASKS_CACHE_DIR_NAME, task_id
+    )
+    task_cache_directory_existed = task_cache_directory.exists()
     try:
         task = _get_task_description(task_id)
         dataset = get_dataset(task.dataset_id, **get_dataset_kwargs)
@@ -431,8 +432,8 @@ def get_task(
         if download_splits and isinstance(task, OpenMLSupervisedTask):
             task.download_split()
     except Exception as e:
-        if not tid_cache_dir_existed:
-            openml.utils._remove_cache_dir_for_id(TASKS_CACHE_DIR_NAME, tid_cache_dir)
+        if not task_cache_directory_existed:
+            openml.utils._remove_cache_dir_for_id(TASKS_CACHE_DIR_NAME, task_cache_directory)
         raise e
 
     return task

openml/testing.py

@@ -47,7 +47,7 @@ class TestBase(unittest.TestCase):
         "user": [],
     }
     flow_name_tracker: ClassVar[list[str]] = []
-    test_server = "https://test.openml.org/api/v1/xml"
+    test_server = f"{openml.config.TEST_SERVER_URL}/api/v1/xml"
     admin_key = "abc"
     user_key = openml.config._TEST_SERVER_NORMAL_USER_KEY
 

tests/conftest.py

@@ -277,7 +277,7 @@ def with_server(request):
         openml.config.apikey = None
         yield
         return
-    openml.config.server = "https://test.openml.org/api/v1/xml"
+    openml.config.server = f"{openml.config.TEST_SERVER_URL}/api/v1/xml"
     openml.config.apikey = TestBase.user_key
     yield
 
@@ -295,8 +295,8 @@ def with_test_cache(test_files_directory, request):
     openml.config.set_root_cache_directory(_root_cache_directory)
     if tmp_cache.exists():
         shutil.rmtree(tmp_cache)
-
 
+
 @pytest.fixture
 def static_cache_dir():
     return Path(__file__).parent / "files" 

tests/test_datasets/test_dataset_functions.py

@@ -527,27 +527,20 @@ def test_deletion_of_cache_dir(self):
     def test_deletion_of_cache_dir_faulty_download(self, patch):
         patch.side_effect = Exception("Boom!")
         self.assertRaisesRegex(Exception, "Boom!", openml.datasets.get_dataset, dataset_id=1)
-        datasets_cache_dir = os.path.join(self.workdir, "org", "openml", "test", "datasets")
+        datasets_cache_dir = os.path.join(openml.config.get_cache_directory(), "datasets")
         assert len(os.listdir(datasets_cache_dir)) == 0
 
     @pytest.mark.uses_test_server()
     def test_publish_dataset(self):
-        # lazy loading not possible as we need the arff-file.
-        openml.datasets.get_dataset(3, download_data=True)
-        file_path = os.path.join(
-            openml.config.get_cache_directory(),
-            "datasets",
-            "3",
-            "dataset.arff",
-        )
+        arff_file_path = self.static_cache_dir / "org" / "openml" / "test" / "datasets" / "2" / "dataset.arff"
         dataset = OpenMLDataset(
             "anneal",
             "test",
             data_format="arff",
             version=1,
             licence="public",
             default_target_attribute="class",
-            data_file=file_path,
+            data_file=arff_file_path,
         )
         dataset.publish()
         TestBase._mark_entity_for_removal("data", dataset.dataset_id)
@@ -886,7 +879,7 @@ def test_create_invalid_dataset(self):
 
     @pytest.mark.uses_test_server()
     def test_get_online_dataset_arff(self):
-        dataset_id = 100  # Australian
+        dataset_id = 128  # iris -- one of the few datasets without parquet file
         # lazy loading not used as arff file is checked.
         dataset = openml.datasets.get_dataset(dataset_id, download_data=True)
         decoder = arff.ArffDecoder()
@@ -1464,8 +1457,9 @@ def test_data_edit_critical_field(self):
                     raise e
                 time.sleep(10)
                 # Delete the cache dir to get the newer version of the dataset
+
                 shutil.rmtree(
-                    os.path.join(self.workdir, "org", "openml", "test", "datasets", str(did)),
+                    os.path.join(openml.config.get_cache_directory(), "datasets", str(did)),
                 )
 
     @pytest.mark.uses_test_server()
@@ -1730,7 +1724,6 @@ def test_delete_dataset(self):
 
 @mock.patch.object(requests.Session, "delete")
 def test_delete_dataset_not_owned(mock_delete, test_files_directory, test_api_key):
-    openml.config.start_using_configuration_for_example()
     content_file = (
         test_files_directory / "mock_responses" / "datasets" / "data_delete_not_owned.xml"
     )
@@ -1745,14 +1738,13 @@ def test_delete_dataset_not_owned(mock_delete, test_files_directory, test_api_ke
     ):
         openml.datasets.delete_dataset(40_000)
 
-    dataset_url = "https://test.openml.org/api/v1/xml/data/40000"
+    dataset_url = f"{openml.config.TEST_SERVER_URL}/api/v1/xml/data/40000"
     assert dataset_url == mock_delete.call_args.args[0]
     assert test_api_key == mock_delete.call_args.kwargs.get("params", {}).get("api_key")
 
 
 @mock.patch.object(requests.Session, "delete")
 def test_delete_dataset_with_run(mock_delete, test_files_directory, test_api_key):
-    openml.config.start_using_configuration_for_example()
     content_file = (
         test_files_directory / "mock_responses" / "datasets" / "data_delete_has_tasks.xml"
     )
@@ -1767,14 +1759,13 @@ def test_delete_dataset_with_run(mock_delete, test_files_directory, test_api_key
     ):
         openml.datasets.delete_dataset(40_000)
 
-    dataset_url = "https://test.openml.org/api/v1/xml/data/40000"
+    dataset_url = f"{openml.config.TEST_SERVER_URL}/api/v1/xml/data/40000"
     assert dataset_url == mock_delete.call_args.args[0]
     assert test_api_key == mock_delete.call_args.kwargs.get("params", {}).get("api_key")
 
 
 @mock.patch.object(requests.Session, "delete")
 def test_delete_dataset_success(mock_delete, test_files_directory, test_api_key):
-    openml.config.start_using_configuration_for_example()
     content_file = (
         test_files_directory / "mock_responses" / "datasets" / "data_delete_successful.xml"
     )
@@ -1786,14 +1777,13 @@ def test_delete_dataset_success(mock_delete, test_files_directory, test_api_key)
     success = openml.datasets.delete_dataset(40000)
     assert success
 
-    dataset_url = "https://test.openml.org/api/v1/xml/data/40000"
+    dataset_url = f"{openml.config.TEST_SERVER_URL}/api/v1/xml/data/40000"
     assert dataset_url == mock_delete.call_args.args[0]
     assert test_api_key == mock_delete.call_args.kwargs.get("params", {}).get("api_key")
 
 
 @mock.patch.object(requests.Session, "delete")
 def test_delete_unknown_dataset(mock_delete, test_files_directory, test_api_key):
-    openml.config.start_using_configuration_for_example()
     content_file = (
         test_files_directory / "mock_responses" / "datasets" / "data_delete_not_exist.xml"
     )
@@ -1808,7 +1798,7 @@ def test_delete_unknown_dataset(mock_delete, test_files_directory, test_api_key)
     ):
         openml.datasets.delete_dataset(9_999_999)
 
-    dataset_url = "https://test.openml.org/api/v1/xml/data/9999999"
+    dataset_url = f"{openml.config.TEST_SERVER_URL}/api/v1/xml/data/9999999"
     assert dataset_url == mock_delete.call_args.args[0]
     assert test_api_key == mock_delete.call_args.kwargs.get("params", {}).get("api_key")
 
@@ -1903,9 +1893,8 @@ def _dataset_features_is_downloaded(did: int):
 
 
 def _dataset_data_file_is_downloaded(did: int):
-    parquet_present = _dataset_file_is_downloaded(did, "dataset.pq")
-    arff_present = _dataset_file_is_downloaded(did, "dataset.arff")
-    return parquet_present or arff_present
+    cache_directory = Path(openml.config.get_cache_directory()) / "datasets" / str(did)
+    return any(f.suffix in (".pq", ".arff") for f in cache_directory.iterdir())
 
 
 def _assert_datasets_retrieved_successfully(
@@ -2010,7 +1999,7 @@ def test_get_dataset_parquet(requests_mock, test_files_directory):
             test_files_directory / "mock_responses" / "datasets" / "data_description_61.xml"
     )
     # While the mocked example is from production, unit tests by default connect to the test server.
-    requests_mock.get("https://test.openml.org/api/v1/xml/data/61", text=content_file.read_text())
+    requests_mock.get(f"{openml.config.TEST_SERVER_URL}/api/v1/xml/data/61", text=content_file.read_text())
     dataset = openml.datasets.get_dataset(61, download_data=True)
     assert dataset._parquet_url is not None
     assert dataset.parquet_file is not None