diff --git a/.github/workflows/e2e-subtensor-tests.yml b/.github/workflows/e2e-subtensor-tests.yml index 9d2e97cc1..b4dc3120f 100644 --- a/.github/workflows/e2e-subtensor-tests.yml +++ b/.github/workflows/e2e-subtensor-tests.yml @@ -24,48 +24,81 @@ env: VERBOSE: ${{ github.event.inputs.verbose }} jobs: - run-tests: - runs-on: SubtensorCI - if: ${{ github.event_name != 'pull_request' || github.event.pull_request.draft == false }} - timeout-minutes: 180 - env: - RELEASE_NAME: development - RUSTV: stable - RUST_BACKTRACE: full - RUST_BIN_DIR: target/x86_64-unknown-linux-gnu - TARGET: x86_64-unknown-linux-gnu + find-tests: + runs-on: ubuntu-latest + if: ${{ github.event_name != 'pull_request' || github.event.pull_request.draft == false }} + outputs: + test-files: ${{ steps.get-tests.outputs.test-files }} steps: - name: Check-out repository under $GITHUB_WORKSPACE - uses: actions/checkout@v2 + uses: actions/checkout@v4 - - name: Install dependencies + - name: Find test files + id: get-tests run: | - sudo apt-get update && - sudo apt-get install -y clang curl libssl-dev llvm libudev-dev protobuf-compiler + test_files=$(find tests/e2e_tests -name "test*.py" | jq -R -s -c 'split("\n") | map(select(. != ""))') + echo "::set-output name=test-files::$test_files" + shell: bash + + pull-docker-image: + runs-on: ubuntu-latest + steps: + - name: Log in to GitHub Container Registry + run: echo "${{ secrets.GITHUB_TOKEN }}" | docker login ghcr.io -u $GITHUB_ACTOR --password-stdin + + - name: Pull Docker Image + run: docker pull ghcr.io/opentensor/subtensor-localnet:latest + + - name: Save Docker Image to Cache + run: docker save -o subtensor-localnet.tar ghcr.io/opentensor/subtensor-localnet:latest - - name: Install Rust ${{ env.RUSTV }} - uses: actions-rs/toolchain@v1.0.6 + - name: Upload Docker Image as Artifact + uses: actions/upload-artifact@v4 with: - toolchain: ${{ env.RUSTV }} - components: rustfmt - profile: minimal + name: subtensor-localnet + path: subtensor-localnet.tar - - name: Add wasm32-unknown-unknown target - run: | - rustup target add wasm32-unknown-unknown --toolchain stable-x86_64-unknown-linux-gnu - rustup component add rust-src --toolchain stable-x86_64-unknown-linux-gnu + run-e2e-tests: + name: ${{ matrix.test-file }} / Python ${{ matrix.python-version }} + needs: + - find-tests + - pull-docker-image + runs-on: ubuntu-latest + timeout-minutes: 45 + strategy: + fail-fast: false # Allow other matrix jobs to run even if this job fails + max-parallel: 32 # Set the maximum number of parallel jobs (same as we have cores in SubtensorCI runner) + matrix: + os: + - ubuntu-latest + test-file: ${{ fromJson(needs.find-tests.outputs.test-files) }} + # python-version: ["3.9", "3.10", "3.11", "3.12", "3.13"] + steps: + - name: Check-out repository + uses: actions/checkout@v4 + + - name: Install uv + uses: astral-sh/setup-uv@v4 + with: + python-version: 3.13 - - name: Clone subtensor repo - run: git clone https://github.com/opentensor/subtensor.git + - name: install dependencies + run: | + uv venv .venv + source .venv/bin/activate + uv pip install .[dev] + uv pip install pytest - - name: Setup subtensor repo - working-directory: ${{ github.workspace }}/subtensor - run: git checkout devnet-ready + - name: Download Cached Docker Image + uses: actions/download-artifact@v4 + with: + name: subtensor-localnet - - name: Install Python dependencies - run: python3 -m pip install -e . pytest + - name: Load Docker Image + run: docker load -i subtensor-localnet.tar - - name: Run all tests + - name: Run tests run: | - LOCALNET_SH_PATH="${{ github.workspace }}/subtensor/scripts/localnet.sh" pytest tests/e2e_tests -s \ No newline at end of file + source .venv/bin/activate + uv run pytest ${{ matrix.test-file }} -s diff --git a/CHANGELOG.md b/CHANGELOG.md index 18a999205..75397415f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,6 +3,7 @@ ## 9.2.0 /2025-03-18 ## What's Changed +* Improve e2e tests' workflow by @roman-opentensor in https://github.com/opentensor/btcli/pull/393 * Updates to E2E suubtensor tests to devnet ready by @ibraheem-opentensor in https://github.com/opentensor/btcli/pull/390 * Allow Py 3.13 install by @thewhaleking in https://github.com/opentensor/btcli/pull/392 * pip install readme by @thewhaleking in https://github.com/opentensor/btcli/pull/391 diff --git a/pyproject.toml b/pyproject.toml index f9793d4a6..2652c869a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -41,7 +41,6 @@ dependencies = [ [project.optional-dependencies] cuda = [ "torch>=1.13.1,<2.6.0", - "cubit>=1.1.0" ] [project.urls] diff --git a/tests/e2e_tests/conftest.py b/tests/e2e_tests/conftest.py index 4d9f9c7b1..e3625683f 100644 --- a/tests/e2e_tests/conftest.py +++ b/tests/e2e_tests/conftest.py @@ -5,6 +5,7 @@ import shutil import signal import subprocess +import sys import time import pytest @@ -13,9 +14,48 @@ from .utils import setup_wallet +def wait_for_node_start(process, pattern, timestamp: int = None): + for line in process.stdout: + print(line.strip()) + # 20 min as timeout + timestamp = timestamp or int(time.time()) + if int(time.time()) - timestamp > 20 * 60: + pytest.fail("Subtensor not started in time") + if pattern.search(line): + print("Node started!") + break + + # Fixture for setting up and tearing down a localnet.sh chain between tests @pytest.fixture(scope="function") def local_chain(request): + """Determines whether to run the localnet.sh script in a subprocess or a Docker container.""" + args = request.param if hasattr(request, "param") else None + params = "" if args is None else f"{args}" + if shutil.which("docker") and not os.getenv("USE_DOCKER") == "0": + yield from docker_runner(params) + else: + if not os.getenv("USE_DOCKER") == "0": + if sys.platform.startswith("linux"): + docker_command = ( + "Install docker with command " + "[blue]sudo apt-get update && sudo apt-get install docker.io -y[/blue]" + " or use documentation [blue]https://docs.docker.com/engine/install/[/blue]" + ) + elif sys.platform == "darwin": + docker_command = ( + "Install docker with command [blue]brew install docker[/blue]" + ) + else: + docker_command = "[blue]Unknown OS, install Docker manually: https://docs.docker.com/get-docker/[/blue]" + + logging.warning("Docker not found in the operating system!") + logging.warning(docker_command) + logging.warning("Tests are run in legacy mode.") + yield from legacy_runner(request) + + +def legacy_runner(request): param = request.param if hasattr(request, "param") else None # Get the environment variable for the script path script_path = os.getenv("LOCALNET_SH_PATH") @@ -41,18 +81,6 @@ def local_chain(request): # Install neuron templates logging.info("Downloading and installing neuron templates from github") - timestamp = int(time.time()) - - def wait_for_node_start(process, pattern): - for line in process.stdout: - print(line.strip()) - # 20 min as timeout - if int(time.time()) - timestamp > 20 * 60: - pytest.fail("Subtensor not started in time") - if pattern.search(line): - print("Node started!") - break - wait_for_node_start(process, pattern) # Run the test, passing in substrate interface @@ -72,6 +100,108 @@ def wait_for_node_start(process, pattern): process.wait() +def docker_runner(params): + """Starts a Docker container before tests and gracefully terminates it after.""" + + def is_docker_running(): + """Check if Docker has been run.""" + try: + subprocess.run( + ["docker", "info"], + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, + check=True, + ) + return True + except subprocess.CalledProcessError: + return False + + def try_start_docker(): + """Run docker based on OS.""" + try: + subprocess.run(["open", "-a", "Docker"], check=True) # macOS + except (FileNotFoundError, subprocess.CalledProcessError): + try: + subprocess.run(["systemctl", "start", "docker"], check=True) # Linux + except (FileNotFoundError, subprocess.CalledProcessError): + try: + subprocess.run( + ["sudo", "service", "docker", "start"], check=True + ) # Linux alternative + except (FileNotFoundError, subprocess.CalledProcessError): + print("Failed to start Docker. Manual start may be required.") + return False + + # Wait Docker run 10 attempts with 3 sec waits + for _ in range(10): + if is_docker_running(): + return True + time.sleep(3) + + print("Docker wasn't run. Manual start may be required.") + return False + + container_name = f"test_local_chain_{str(time.time()).replace('.', '_')}" + image_name = "ghcr.io/opentensor/subtensor-localnet:latest" + + # Command to start container + cmds = [ + "docker", + "run", + "--rm", + "--name", + container_name, + "-p", + "9944:9944", + "-p", + "9945:9945", + image_name, + params, + ] + + try_start_docker() + + # Start container + with subprocess.Popen( + cmds, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + start_new_session=True, + ) as process: + try: + substrate = None + try: + pattern = re.compile(r"Imported #1") + wait_for_node_start(process, pattern, int(time.time())) + except TimeoutError: + raise + + result = subprocess.run( + ["docker", "ps", "-q", "-f", f"name={container_name}"], + capture_output=True, + text=True, + ) + if not result.stdout.strip(): + raise RuntimeError("Docker container failed to start.") + + substrate = AsyncSubstrateInterface(url="ws://127.0.0.1:9944") + yield substrate + + finally: + try: + if substrate: + substrate.close() + except Exception: + pass + + try: + subprocess.run(["docker", "kill", container_name]) + process.wait(timeout=10) + except subprocess.TimeoutExpired: + os.killpg(os.getpgid(process.pid), signal.SIGKILL) + + @pytest.fixture(scope="function") def wallet_setup(): wallet_paths = [] diff --git a/tests/e2e_tests/test_senate.py b/tests/e2e_tests/test_senate.py index 7d8a52418..dfd2cfa57 100644 --- a/tests/e2e_tests/test_senate.py +++ b/tests/e2e_tests/test_senate.py @@ -216,17 +216,17 @@ def test_senate(local_chain, wallet_setup): proposals_after_nay_output = proposals_after_nay.stdout.splitlines() # Total Ayes to remain 1 - proposals_after_nay_output[9].split()[2] == "1" + assert proposals_after_nay_output[9].split()[2] == "1" # Total Nays increased to 1 - proposals_after_nay_output[9].split()[4] == "1" + assert proposals_after_nay_output[9].split()[4] == "1" # Assert Alice has voted Nay - proposals_after_nay_output[10].split()[0].strip( + assert proposals_after_nay_output[10].split()[0].strip( ":" ) == wallet_alice.hotkey.ss58_address # Assert vote casted as Nay - proposals_after_nay_output[9].split()[1] == "Nay" + assert proposals_after_nay_output[10].split()[1] == "Nay" print("✅ Passed senate commands") diff --git a/tests/e2e_tests/test_staking_sudo.py b/tests/e2e_tests/test_staking_sudo.py index c1b939c8f..f8bfc6d0c 100644 --- a/tests/e2e_tests/test_staking_sudo.py +++ b/tests/e2e_tests/test_staking_sudo.py @@ -1,5 +1,4 @@ import re -import time from bittensor_cli.src.bittensor.balances import Balance diff --git a/tests/e2e_tests/test_wallet_creations.py b/tests/e2e_tests/test_wallet_creations.py index ac82ec2a4..ea0959985 100644 --- a/tests/e2e_tests/test_wallet_creations.py +++ b/tests/e2e_tests/test_wallet_creations.py @@ -486,7 +486,7 @@ def test_wallet_balance_all(local_chain, wallet_setup, capfd): wallet_name = f"test_wallet_{i}" wallet_names.append(wallet_name) - result = exec_command( + exec_command( command="wallet", sub_command="new-coldkey", extra_args=[