diff --git a/.github/workflows/integration-tests.yml b/.github/workflows/integration-tests.yml index d1b63ce56..9bd073cc2 100644 --- a/.github/workflows/integration-tests.yml +++ b/.github/workflows/integration-tests.yml @@ -1,3 +1,27 @@ +# OpenHCS Integration Tests +# +# CI Strategy: Optimized boundary testing (11 jobs total) +# +# Group 1: Python boundary tests (4 jobs) +# - Python 3.11 (oldest) and 3.13 (newest well-supported) on Linux + Windows +# - Python 3.14 excluded: no pre-built NumPy/zeroc-ice wheels yet +# +# Group 2: Backend/microscope coverage (4 jobs) +# - Python 3.12 on Linux + Windows +# - All backends: disk, zarr +# - All microscopes: ImageXpress, OperaPhenix +# +# Group 3: OMERO tests (2 jobs) +# - Python 3.11 and 3.12 on Linux only +# - Uses pre-built zeroc-ice wheels (no Python 3.13+ support yet) +# +# Group 4: Wheel installation test (1 job) +# - Python 3.12 on Linux +# - Validates PyPI-style installation +# +# Total: 11 jobs covering all Python versions (3.11-3.13), both OSes, +# all backends, and all microscope formats + name: Integration Tests on: @@ -17,109 +41,127 @@ on: - full-coverage jobs: - integration-tests: - runs-on: ubuntu-latest + # Group 1: Python version boundary testing (4 jobs) + # Test oldest (3.11) and newest (3.13) Python on both OSes + # Python 3.14 excluded: no pre-built NumPy wheels yet (builds from source stall on Windows) + python-boundary-tests: + runs-on: ${{ matrix.os }} + if: github.event_name == 'push' || github.event_name == 'pull_request' strategy: + fail-fast: false matrix: - backend: [disk, zarr] - microscope: [ImageXpress, OperaPhenix] - + # Boundary testing: oldest and newest well-supported versions + python-version: ["3.11", "3.13"] + os: [ubuntu-latest, windows-latest] + steps: - name: Checkout uses: actions/checkout@v4 - - name: Setup Python + - name: Setup Python ${{ matrix.python-version }} uses: actions/setup-python@v5 with: - python-version: "3.12" + python-version: ${{ matrix.python-version }} - name: Install dependencies run: | python -m pip install --upgrade pip pip install -e ".[dev]" - - name: Run CPU-only integration tests - if: github.event.inputs.test_mode == 'cpu-only' || github.event.inputs.test_mode == '' + - name: Run boundary version tests env: OPENHCS_CPU_ONLY: true run: | - python -m pytest tests/integration/ \ - --it-backends ${{ matrix.backend }} \ - --it-microscopes ${{ matrix.microscope }} \ - --it-dims 3d \ - --it-exec-mode multiprocessing \ - -v --tb=short - - - name: Run full coverage integration tests - if: github.event.inputs.test_mode == 'full-coverage' - run: | - python -m pytest tests/integration/ \ - --it-backends ${{ matrix.backend }} \ - --it-microscopes ${{ matrix.microscope }} \ - --it-dims all \ - --it-exec-mode all \ - -v --tb=short + python -m pytest tests/integration/ --it-backends disk --it-microscopes ImageXpress --it-dims 3d --it-exec-mode multiprocessing -v --tb=short - integration-tests-focused: - runs-on: ubuntu-latest + # Group 2: Backend and microscope coverage (4 jobs) + # Test all backend/microscope combinations on both OSes with Python 3.12 + backend-microscope-tests: + runs-on: ${{ matrix.os }} if: github.event_name == 'push' || github.event_name == 'pull_request' strategy: + fail-fast: false matrix: - python-version: ["3.11", "3.12"] + os: [ubuntu-latest, windows-latest] + backend: [disk, zarr] + microscope: [ImageXpress, OperaPhenix] steps: - name: Checkout uses: actions/checkout@v4 - - name: Setup Python ${{ matrix.python-version }} + - name: Setup Python 3.12 uses: actions/setup-python@v5 with: - python-version: ${{ matrix.python-version }} + python-version: "3.12" - name: Install dependencies run: | python -m pip install --upgrade pip pip install -e ".[dev]" - - name: Run focused CPU-only integration tests (default 4 combinations) + - name: Run backend/microscope combination tests env: OPENHCS_CPU_ONLY: true run: | - python -m pytest tests/integration/ \ - --it-backends disk,zarr \ - --it-microscopes ImageXpress,OperaPhenix \ - --it-dims 3d \ - --it-exec-mode multiprocessing \ - -v --tb=short + python -m pytest tests/integration/ --it-backends ${{ matrix.backend }} --it-microscopes ${{ matrix.microscope }} --it-dims 3d --it-exec-mode multiprocessing -v --tb=short - # Windows integration tests to catch path separator issues - integration-tests-windows: - runs-on: windows-latest + # Group 3: OMERO testing on Linux (2 jobs) + # Test OMERO on Linux with Python 3.11 and 3.12 + # Python 3.13+ not supported yet - zeroc-ice pre-built wheels only available up to Python 3.12 + # See: https://github.com/glencoesoftware/zeroc-ice-py-linux-x86_64/releases/tag/20240202 + omero-tests-linux: + runs-on: ubuntu-latest if: github.event_name == 'push' || github.event_name == 'pull_request' strategy: + fail-fast: false matrix: - backend: [disk, zarr] - microscope: [ImageXpress, OperaPhenix] + python-version: ["3.11", "3.12"] + include: + - python-version: "3.11" + ice-wheel: "https://github.com/glencoesoftware/zeroc-ice-py-linux-x86_64/releases/download/20240202/zeroc_ice-3.6.5-cp311-cp311-manylinux_2_28_x86_64.whl" + - python-version: "3.12" + ice-wheel: "https://github.com/glencoesoftware/zeroc-ice-py-linux-x86_64/releases/download/20240202/zeroc_ice-3.6.5-cp312-cp312-manylinux_2_28_x86_64.whl" steps: - name: Checkout uses: actions/checkout@v4 - - name: Setup Python + - name: Setup Python ${{ matrix.python-version }} uses: actions/setup-python@v5 with: - python-version: "3.12" + python-version: ${{ matrix.python-version }} - name: Install dependencies run: | python -m pip install --upgrade pip - pip install -e ".[dev]" + # Install pre-built zeroc-ice wheel from Glencoe Software (OMERO maintainers) + # Building from source fails on Python 3.11+ due to setuptools compatibility issues + pip install ${{ matrix.ice-wheel }} + pip install -e ".[dev,omero]" - - name: Run Windows CPU-only integration tests + - name: Run OMERO integration tests (auto-starts Docker + OMERO) env: OPENHCS_CPU_ONLY: true run: | - python -m pytest tests/integration/ --it-backends ${{ matrix.backend }} --it-microscopes ${{ matrix.microscope }} --it-dims 3d --it-exec-mode multiprocessing -v --tb=short + python -m pytest tests/integration/ --it-backends disk --it-microscopes OMERO --it-dims 3d --it-exec-mode multiprocessing --it-visualizers none -v --tb=short -s --log-cli-level=INFO + timeout-minutes: 15 + + - name: Show OMERO logs on failure + if: failure() + run: | + echo "=== OMERO Server Logs ===" + sudo docker compose -f openhcs/omero/docker-compose.yml logs omeroserver || true + echo "=== OMERO Web Logs ===" + sudo docker compose -f openhcs/omero/docker-compose.yml logs omeroweb || true + echo "=== Database Logs ===" + sudo docker compose -f openhcs/omero/docker-compose.yml logs database || true + + - name: Cleanup OMERO containers + if: always() + run: | + cd openhcs/omero + sudo docker compose down -v || true # Code quality checks (linting, formatting, type checking) code-quality: @@ -248,51 +290,4 @@ jobs: deactivate rm -rf test_wheel - omero-integration-tests: - runs-on: ubuntu-latest - if: github.event_name == 'push' || github.event_name == 'pull_request' - steps: - - name: Checkout - uses: actions/checkout@v4 - - - name: Setup Python - uses: actions/setup-python@v5 - with: - python-version: "3.11" - - - name: Install dependencies - run: | - python -m pip install --upgrade pip - # Install pre-built zeroc-ice wheel from Glencoe Software - pip install https://github.com/glencoesoftware/zeroc-ice-py-linux-x86_64/releases/download/20240202/zeroc_ice-3.6.5-cp311-cp311-manylinux_2_28_x86_64.whl - pip install -e ".[dev,omero]" - - - name: Run OMERO integration tests (auto-starts Docker + OMERO) - env: - OPENHCS_CPU_ONLY: true - run: | - python -m pytest tests/integration/ \ - --it-backends disk \ - --it-microscopes OMERO \ - --it-dims 3d \ - --it-exec-mode multiprocessing \ - --it-visualizers none \ - -v --tb=short -s --log-cli-level=INFO - timeout-minutes: 15 - - - name: Show OMERO logs on failure - if: failure() - run: | - echo "=== OMERO Server Logs ===" - sudo docker compose -f openhcs/omero/docker-compose.yml logs omeroserver || true - echo "=== OMERO Web Logs ===" - sudo docker compose -f openhcs/omero/docker-compose.yml logs omeroweb || true - echo "=== Database Logs ===" - sudo docker compose -f openhcs/omero/docker-compose.yml logs database || true - - - name: Cleanup OMERO containers - if: always() - run: | - cd openhcs/omero - sudo docker compose down -v || true diff --git a/openhcs/core/pipeline/path_planner.py b/openhcs/core/pipeline/path_planner.py index 7829c65e7..ae4eadf85 100644 --- a/openhcs/core/pipeline/path_planner.py +++ b/openhcs/core/pipeline/path_planner.py @@ -473,9 +473,12 @@ def build_dict_pattern_path(base_path: str, dict_key: str) -> str: Returns: Channel-specific path """ - dir_part, filename = base_path.rsplit('/', 1) + # Use Path for cross-platform path handling (Windows uses backslashes) + path = Path(base_path) + dir_part = path.parent + filename = path.name well_id, rest = filename.split('_', 1) - return f"{dir_part}/{well_id}_w{dict_key}_{rest}" + return str(dir_part / f"{well_id}_w{dict_key}_{rest}") diff --git a/openhcs/io/atomic.py b/openhcs/io/atomic.py index 7f23d0693..aa96f9c50 100644 --- a/openhcs/io/atomic.py +++ b/openhcs/io/atomic.py @@ -136,7 +136,9 @@ def atomic_write_json( try: tmp_path = _write_to_temp_file(file_path, data, indent) - os.rename(tmp_path, str(file_path)) + # Use os.replace() instead of os.rename() for atomic replacement on all platforms + # os.rename() fails on Windows if destination exists, os.replace() works on both Unix and Windows + os.replace(tmp_path, str(file_path)) logger.debug(f"Atomically wrote JSON to {file_path}") except Exception as e: raise FileLockError(f"Atomic JSON write failed for {file_path}: {e}") from e diff --git a/openhcs/io/omero_local.py b/openhcs/io/omero_local.py index 7ebd9b219..f210a90e2 100644 --- a/openhcs/io/omero_local.py +++ b/openhcs/io/omero_local.py @@ -595,7 +595,9 @@ def _save_csv_as_table(self, csv_content: str, output_path: Path, images_dir: st raise ValueError(f"Plate '{plate_name}' not found in OMERO (images dir: {images_dir})") # Determine table name from filename - table_name = output_path.stem + # Remove ALL extensions (e.g., "file.roi.zip.json" -> "file") + # OMERO table names cannot contain dots except for the .h5 extension + table_name = output_path.name.split('.')[0] # Build column objects based on DataFrame dtypes columns = [] diff --git a/openhcs/microscopes/imagexpress.py b/openhcs/microscopes/imagexpress.py index 692d9401c..5d0d813ce 100644 --- a/openhcs/microscopes/imagexpress.py +++ b/openhcs/microscopes/imagexpress.py @@ -320,10 +320,11 @@ def _flatten_indexed_folders(self, directory: Path, fm: FileManager, new_filename = self.parser.construct_filename(**metadata) # Build PLATE-RELATIVE virtual flattened path - virtual_relative = str(directory.relative_to(plate_path) / new_filename) + # Use .as_posix() to ensure forward slashes on all platforms (Windows uses backslashes with str()) + virtual_relative = (directory.relative_to(plate_path) / new_filename).as_posix() # Build PLATE-RELATIVE real path (in subfolder) - real_relative = str(Path(img_file).relative_to(plate_path)) + real_relative = Path(img_file).relative_to(plate_path).as_posix() # Add to mapping (both plate-relative) mapping_dict[virtual_relative] = real_relative diff --git a/openhcs/microscopes/opera_phenix.py b/openhcs/microscopes/opera_phenix.py index 5f470792d..f7a61d13d 100644 --- a/openhcs/microscopes/opera_phenix.py +++ b/openhcs/microscopes/opera_phenix.py @@ -156,8 +156,9 @@ def _build_virtual_mapping(self, plate_path: Path, filemanager: FileManager) -> new_name = self.parser.construct_filename(**metadata) # Build PLATE-RELATIVE mapping (no workspace directory) - virtual_relative = str(Path("Images") / new_name) - real_relative = str(Path("Images") / file_name) + # Use .as_posix() to ensure forward slashes on all platforms (Windows uses backslashes with str()) + virtual_relative = (Path("Images") / new_name).as_posix() + real_relative = (Path("Images") / file_name).as_posix() workspace_mapping[virtual_relative] = real_relative logger.info(f"Built {len(workspace_mapping)} virtual path mappings for Opera Phenix")