Improve e2e tests' workflow

.github/workflows/e2e-subtensor-tests.yml

@@ -24,48 +24,81 @@ env:
   VERBOSE: ${{ github.event.inputs.verbose }}
 
 jobs:
-  run-tests:
-    runs-on: SubtensorCI
-    if: ${{ github.event_name != 'pull_request' || github.event.pull_request.draft == false }}
-    timeout-minutes: 180
-    env:
-      RELEASE_NAME: development
-      RUSTV: stable
-      RUST_BACKTRACE: full
-      RUST_BIN_DIR: target/x86_64-unknown-linux-gnu
-      TARGET: x86_64-unknown-linux-gnu
 
+  find-tests:
+    runs-on: ubuntu-latest
+    if: ${{ github.event_name != 'pull_request' || github.event.pull_request.draft == false }}
+    outputs:
+      test-files: ${{ steps.get-tests.outputs.test-files }}
     steps:
       - name: Check-out repository under $GITHUB_WORKSPACE
-        uses: actions/checkout@v2
+        uses: actions/checkout@v4
 
-      - name: Install dependencies
+      - name: Find test files
+        id: get-tests
         run: |
-          sudo apt-get update &&
-          sudo apt-get install -y clang curl libssl-dev llvm libudev-dev protobuf-compiler
+          test_files=$(find tests/e2e_tests -name "test*.py" | jq -R -s -c 'split("\n") | map(select(. != ""))')
+          echo "::set-output name=test-files::$test_files"
+        shell: bash
+
+  pull-docker-image:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Log in to GitHub Container Registry
+        run: echo "${{ secrets.GITHUB_TOKEN }}" | docker login ghcr.io -u $GITHUB_ACTOR --password-stdin
+
+      - name: Pull Docker Image
+        run: docker pull ghcr.io/opentensor/subtensor-localnet:latest
+
+      - name: Save Docker Image to Cache
+        run: docker save -o subtensor-localnet.tar ghcr.io/opentensor/subtensor-localnet:latest
 
-      - name: Install Rust ${{ env.RUSTV }}
-        uses: actions-rs/toolchain@v1.0.6
+      - name: Upload Docker Image as Artifact
+        uses: actions/upload-artifact@v4
         with:
-          toolchain: ${{ env.RUSTV }}
-          components: rustfmt
-          profile: minimal
+          name: subtensor-localnet
+          path: subtensor-localnet.tar
 
-      - name: Add wasm32-unknown-unknown target
-        run: |
-          rustup target add wasm32-unknown-unknown --toolchain stable-x86_64-unknown-linux-gnu
-          rustup component add rust-src --toolchain stable-x86_64-unknown-linux-gnu
+  run-e2e-tests:
+    name: ${{ matrix.test-file }} / Python ${{ matrix.python-version }}
+    needs:
+      - find-tests
+      - pull-docker-image
+    runs-on: ubuntu-latest
+    timeout-minutes: 45
+    strategy:
+      fail-fast: false  # Allow other matrix jobs to run even if this job fails
+      max-parallel: 32  # Set the maximum number of parallel jobs (same as we have cores in SubtensorCI runner)
+      matrix:
+        os:
+          - ubuntu-latest
+        test-file: ${{ fromJson(needs.find-tests.outputs.test-files) }}
+        # python-version: ["3.9", "3.10", "3.11", "3.12", "3.13"]
+    steps:
+      - name: Check-out repository
+        uses: actions/checkout@v4
+
+      - name: Install uv
+        uses: astral-sh/setup-uv@v4
+        with:
+          python-version: 3.13
 
-      - name: Clone subtensor repo
-        run: git clone https://github.com/opentensor/subtensor.git
+      - name: install dependencies
+        run: |
+          uv venv .venv
+          source .venv/bin/activate
+          uv pip install .[dev]
+          uv pip install pytest
 
-      - name: Setup subtensor repo
-        working-directory: ${{ github.workspace }}/subtensor
-        run: git checkout devnet-ready
+      - name: Download Cached Docker Image
+        uses: actions/download-artifact@v4
+        with:
+          name: subtensor-localnet
 
-      - name: Install Python dependencies
-        run: python3 -m pip install -e . pytest
+      - name: Load Docker Image
+        run: docker load -i subtensor-localnet.tar
 
-      - name: Run all tests
+      - name: Run tests
         run: |
-          LOCALNET_SH_PATH="${{ github.workspace }}/subtensor/scripts/localnet.sh" pytest tests/e2e_tests -s
+          source .venv/bin/activate
+          uv run pytest ${{ matrix.test-file }} -s

CHANGELOG.md

@@ -3,6 +3,7 @@
 ## 9.2.0 /2025-03-18
 
 ## What's Changed
+* Improve e2e tests' workflow by @roman-opentensor in https://github.com/opentensor/btcli/pull/393
 * Updates to E2E suubtensor tests to devnet ready by @ibraheem-opentensor in https://github.com/opentensor/btcli/pull/390
 * Allow Py 3.13 install by @thewhaleking in https://github.com/opentensor/btcli/pull/392
 * pip install readme by @thewhaleking in https://github.com/opentensor/btcli/pull/391

pyproject.toml

@@ -41,7 +41,6 @@ dependencies = [
 [project.optional-dependencies]
 cuda = [
     "torch>=1.13.1,<2.6.0",
-    "cubit>=1.1.0"
 ]
 
 [project.urls]

tests/e2e_tests/conftest.py

@@ -5,6 +5,7 @@
 import shutil
 import signal
 import subprocess
+import sys
 import time
 
 import pytest
@@ -13,9 +14,48 @@
 from .utils import setup_wallet
 
 
+def wait_for_node_start(process, pattern, timestamp: int = None):
+    for line in process.stdout:
+        print(line.strip())
+        # 20 min as timeout
+        timestamp = timestamp or int(time.time())
+        if int(time.time()) - timestamp > 20 * 60:
+            pytest.fail("Subtensor not started in time")
+        if pattern.search(line):
+            print("Node started!")
+            break
+
+
 # Fixture for setting up and tearing down a localnet.sh chain between tests
 @pytest.fixture(scope="function")
 def local_chain(request):
+    """Determines whether to run the localnet.sh script in a subprocess or a Docker container."""
+    args = request.param if hasattr(request, "param") else None
+    params = "" if args is None else f"{args}"
+    if shutil.which("docker") and not os.getenv("USE_DOCKER") == "0":
+        yield from docker_runner(params)
+    else:
+        if not os.getenv("USE_DOCKER") == "0":
+            if sys.platform.startswith("linux"):
+                docker_command = (
+                    "Install docker with command "
+                    "[blue]sudo apt-get update && sudo apt-get install docker.io -y[/blue]"
+                    " or use documentation [blue]https://docs.docker.com/engine/install/[/blue]"
+                )
+            elif sys.platform == "darwin":
+                docker_command = (
+                    "Install docker with command [blue]brew install docker[/blue]"
+                )
+            else:
+                docker_command = "[blue]Unknown OS, install Docker manually: https://docs.docker.com/get-docker/[/blue]"
+
+            logging.warning("Docker not found in the operating system!")
+            logging.warning(docker_command)
+            logging.warning("Tests are run in legacy mode.")
+        yield from legacy_runner(request)
+
+
+def legacy_runner(request):
     param = request.param if hasattr(request, "param") else None
     # Get the environment variable for the script path
     script_path = os.getenv("LOCALNET_SH_PATH")
@@ -41,18 +81,6 @@ def local_chain(request):
     # Install neuron templates
     logging.info("Downloading and installing neuron templates from github")
 
-    timestamp = int(time.time())
-
-    def wait_for_node_start(process, pattern):
-        for line in process.stdout:
-            print(line.strip())
-            # 20 min as timeout
-            if int(time.time()) - timestamp > 20 * 60:
-                pytest.fail("Subtensor not started in time")
-            if pattern.search(line):
-                print("Node started!")
-                break
-
     wait_for_node_start(process, pattern)
 
     # Run the test, passing in substrate interface
@@ -72,6 +100,108 @@ def wait_for_node_start(process, pattern):
     process.wait()
 
 
+def docker_runner(params):
+    """Starts a Docker container before tests and gracefully terminates it after."""
+
+    def is_docker_running():
+        """Check if Docker has been run."""
+        try:
+            subprocess.run(
+                ["docker", "info"],
+                stdout=subprocess.DEVNULL,
+                stderr=subprocess.DEVNULL,
+                check=True,
+            )
+            return True
+        except subprocess.CalledProcessError:
+            return False
+
+    def try_start_docker():
+        """Run docker based on OS."""
+        try:
+            subprocess.run(["open", "-a", "Docker"], check=True)  # macOS
+        except (FileNotFoundError, subprocess.CalledProcessError):
+            try:
+                subprocess.run(["systemctl", "start", "docker"], check=True)  # Linux
+            except (FileNotFoundError, subprocess.CalledProcessError):
+                try:
+                    subprocess.run(
+                        ["sudo", "service", "docker", "start"], check=True
+                    )  # Linux alternative
+                except (FileNotFoundError, subprocess.CalledProcessError):
+                    print("Failed to start Docker. Manual start may be required.")
+                    return False
+
+        # Wait Docker run 10 attempts with 3 sec waits
+        for _ in range(10):
+            if is_docker_running():
+                return True
+            time.sleep(3)
+
+        print("Docker wasn't run. Manual start may be required.")
+        return False
+
+    container_name = f"test_local_chain_{str(time.time()).replace('.', '_')}"
+    image_name = "ghcr.io/opentensor/subtensor-localnet:latest"
+
+    # Command to start container
+    cmds = [
+        "docker",
+        "run",
+        "--rm",
+        "--name",
+        container_name,
+        "-p",
+        "9944:9944",
+        "-p",
+        "9945:9945",
+        image_name,
+        params,
+    ]
+
+    try_start_docker()
+
+    # Start container
+    with subprocess.Popen(
+        cmds,
+        stdout=subprocess.PIPE,
+        stderr=subprocess.PIPE,
+        text=True,
+        start_new_session=True,
+    ) as process:
+        try:
+            substrate = None
+            try:
+                pattern = re.compile(r"Imported #1")
+                wait_for_node_start(process, pattern, int(time.time()))
+            except TimeoutError:
+                raise
+
+            result = subprocess.run(
+                ["docker", "ps", "-q", "-f", f"name={container_name}"],
+                capture_output=True,
+                text=True,
+            )
+            if not result.stdout.strip():
+                raise RuntimeError("Docker container failed to start.")
+
+            substrate = AsyncSubstrateInterface(url="ws://127.0.0.1:9944")
+            yield substrate
+
+        finally:
+            try:
+                if substrate:
+                    substrate.close()
+            except Exception:
+                pass
+
+            try:
+                subprocess.run(["docker", "kill", container_name])
+                process.wait(timeout=10)
+            except subprocess.TimeoutExpired:
+                os.killpg(os.getpgid(process.pid), signal.SIGKILL)
+
+
 @pytest.fixture(scope="function")
 def wallet_setup():
     wallet_paths = []

tests/e2e_tests/test_senate.py

@@ -216,17 +216,17 @@ def test_senate(local_chain, wallet_setup):
     proposals_after_nay_output = proposals_after_nay.stdout.splitlines()
 
     # Total Ayes to remain 1
-    proposals_after_nay_output[9].split()[2] == "1"
+    assert proposals_after_nay_output[9].split()[2] == "1"
 
     # Total Nays increased to 1
-    proposals_after_nay_output[9].split()[4] == "1"
+    assert proposals_after_nay_output[9].split()[4] == "1"
 
     # Assert Alice has voted Nay
-    proposals_after_nay_output[10].split()[0].strip(
+    assert proposals_after_nay_output[10].split()[0].strip(
         ":"
     ) == wallet_alice.hotkey.ss58_address
 
     # Assert vote casted as Nay
-    proposals_after_nay_output[9].split()[1] == "Nay"
+    assert proposals_after_nay_output[10].split()[1] == "Nay"
 
     print("✅ Passed senate commands")

tests/e2e_tests/test_staking_sudo.py

@@ -1,5 +1,4 @@
 import re
-import time
 
 from bittensor_cli.src.bittensor.balances import Balance
 

tests/e2e_tests/test_wallet_creations.py

@@ -486,7 +486,7 @@ def test_wallet_balance_all(local_chain, wallet_setup, capfd):
         wallet_name = f"test_wallet_{i}"
         wallet_names.append(wallet_name)
 
-        result = exec_command(
+        exec_command(
             command="wallet",
             sub_command="new-coldkey",
             extra_args=[