diff --git a/.github/workflows/e2e-subtensor-tests.yaml b/.github/workflows/e2e-subtensor-tests.yaml index a95bece4bc..97df3354be 100644 --- a/.github/workflows/e2e-subtensor-tests.yaml +++ b/.github/workflows/e2e-subtensor-tests.yaml @@ -25,7 +25,7 @@ env: # job to run tests in parallel jobs: - # Job to find all test files + find-tests: runs-on: ubuntu-latest if: ${{ github.event_name != 'pull_request' || github.event.pull_request.draft == false }} @@ -42,68 +42,55 @@ jobs: echo "::set-output name=test-files::$test_files" shell: bash + pull-docker-image: + runs-on: ubuntu-latest + steps: + - name: Log in to GitHub Container Registry + run: echo "${{ secrets.GITHUB_TOKEN }}" | docker login ghcr.io -u $GITHUB_ACTOR --password-stdin + + - name: Pull Docker Image + run: docker pull ghcr.io/opentensor/subtensor-localnet:latest + + - name: Save Docker Image to Cache + run: docker save -o subtensor-localnet.tar ghcr.io/opentensor/subtensor-localnet:latest + + - name: Upload Docker Image as Artifact + uses: actions/upload-artifact@v4 + with: + name: subtensor-localnet + path: subtensor-localnet.tar + # Job to run tests in parallel run: - needs: find-tests - runs-on: SubtensorCI + needs: + - find-tests + - pull-docker-image + runs-on: ubuntu-latest timeout-minutes: 45 strategy: fail-fast: false # Allow other matrix jobs to run even if this job fails - max-parallel: 8 # Set the maximum number of parallel jobs + max-parallel: 32 # Set the maximum number of parallel jobs (same as we have cores in SubtensorCI runner) matrix: - rust-branch: - - stable - rust-target: - - x86_64-unknown-linux-gnu os: - ubuntu-latest test-file: ${{ fromJson(needs.find-tests.outputs.test-files) }} - env: - RELEASE_NAME: development - RUSTV: ${{ matrix.rust-branch }} - RUST_BACKTRACE: full - RUST_BIN_DIR: target/${{ matrix.rust-target }} - TARGET: ${{ matrix.rust-target }} steps: - - name: Check-out repository under $GITHUB_WORKSPACE + - name: Check-out repository uses: actions/checkout@v4 - - name: Install dependencies - run: | - sudo apt-get update && - sudo apt-get install -y clang curl libssl-dev llvm libudev-dev protobuf-compiler - - - name: Install Rust ${{ matrix.rust-branch }} - uses: actions-rs/toolchain@v1.0.6 - with: - toolchain: ${{ matrix.rust-branch }} - components: rustfmt - profile: minimal - - - name: Add wasm32-unknown-unknown target - run: | - rustup target add wasm32-unknown-unknown --toolchain stable-x86_64-unknown-linux-gnu - rustup component add rust-src --toolchain stable-x86_64-unknown-linux-gnu - - - name: Clone subtensor repo - run: git clone https://github.com/opentensor/subtensor.git - - - name: Setup subtensor repo - working-directory: ${{ github.workspace }}/subtensor - run: git checkout devnet-ready - - name: Install uv uses: astral-sh/setup-uv@v4 - name: install dependencies run: uv sync --all-extras --dev - - name: Run tests - run: | - LOCALNET_SH_PATH="${{ github.workspace }}/subtensor/scripts/localnet.sh" uv run pytest ${{ matrix.test-file }} -s + - name: Download Cached Docker Image + uses: actions/download-artifact@v4 + with: + name: subtensor-localnet - - name: Retry failed tests - if: failure() - run: | - sleep 10 - LOCALNET_SH_PATH="${{ github.workspace }}/subtensor/scripts/localnet.sh" uv run pytest ${{ matrix.test-file }} -s + - name: Load Docker Image + run: docker load -i subtensor-localnet.tar + + - name: Run tests + run: uv run pytest ${{ matrix.test-file }} -s diff --git a/README.md b/README.md index 109a321030..8de9667c7d 100644 --- a/README.md +++ b/README.md @@ -221,6 +221,49 @@ The Python interpreter output will look like below. >>> ``` +### Testing +You can run integration and unit tests in interactive mode of IDE or in terminal mode using the command: +```bash +pytest tests/integration_tests +pytest tests/unit_tests +``` + +#### E2E tests have 2 options for launching (legacy runner): +- using a compiler based on the substrait code +- using an already built docker image (docker runner) + +#### Using `docker runner` (default for now): +- E2E tests with docker image do not require preliminary compilation +- are executed very quickly +- require docker installed in OS + +Ho to use: +```bash +pytest tests/e2e_tests +``` + +#### TUsing `legacy runner`: +- Will start compilation of the collected code in your subtensor repository +- you must provide the `LOCALNET_SH_PATH` variable in the local environment with the path to the file `/scripts/localnet.sh` in the cloned repository within your OS +- you can use the `BUILD_BINARY=0` variable, this will skip the copy step for each test. +- you can use the `USE_DOCKER=0` variable, this will run tests using the "legacy runner", even if docker is installed in your OS + +#### Ho to use: +Regular e2e tests run +```bash +LOCALNET_SH_PATH=/path/to/your/localnet.sh pytest tests/e2e_tests +``` + +If you want to skip re-build process for each e2e test +```bash +BUILD_BINARY=0 LOCALNET_SH_PATH=/path/to/your/localnet.sh pytest tests/e2e_tests +``` + +If you want to use legacy runner even with installed Docker in your OS +```bash +USE_DOCKER=0 BUILD_BINARY=0 LOCALNET_SH_PATH=/path/to/your/localnet.sh pytest tests/e2e_tests +``` + --- ## Release Guidelines diff --git a/tests/e2e_tests/conftest.py b/tests/e2e_tests/conftest.py index b4770ed053..51a865b1b8 100644 --- a/tests/e2e_tests/conftest.py +++ b/tests/e2e_tests/conftest.py @@ -1,10 +1,12 @@ import os import re import shlex +import shutil import signal import subprocess -import time +import sys import threading +import time import pytest from async_substrate_interface import SubstrateInterface @@ -18,10 +20,66 @@ ) -# Fixture for setting up and tearing down a localnet.sh chain between tests +def wait_for_node_start(process, timestamp=None): + """Waits for node to start in the docker.""" + while True: + line = process.stdout.readline() + if not line: + break + + timestamp = timestamp or int(time.time()) + print(line.strip()) + # 10 min as timeout + if int(time.time()) - timestamp > 20 * 30: + print("Subtensor not started in time") + raise TimeoutError + + pattern = re.compile(r"Imported #1") + if pattern.search(line): + print("Node started!") + break + + # Start a background reader after pattern is found + # To prevent the buffer filling up + def read_output(): + while True: + if not process.stdout.readline(): + break + + reader_thread = threading.Thread(target=read_output, daemon=True) + reader_thread.start() + + @pytest.fixture(scope="function") def local_chain(request): - param = request.param if hasattr(request, "param") else None + """Determines whether to run the localnet.sh script in a subprocess or a Docker container.""" + args = request.param if hasattr(request, "param") else None + params = "" if args is None else f"{args}" + if shutil.which("docker") and not os.getenv("USE_DOCKER") == "0": + yield from docker_runner(params) + else: + if not os.getenv("USE_DOCKER") == "0": + if sys.platform.startswith("linux"): + docker_command = ( + "Install docker with command " + "[blue]sudo apt-get update && sudo apt-get install docker.io -y[/blue]" + " or use documentation [blue]https://docs.docker.com/engine/install/[/blue]" + ) + elif sys.platform == "darwin": + docker_command = ( + "Install docker with command [blue]brew install docker[/blue]" + ) + else: + docker_command = "[blue]Unknown OS, install Docker manually: https://docs.docker.com/get-docker/[/blue]" + + logging.warning("Docker not found in the operating system!") + logging.warning(docker_command) + logging.warning("Tests are run in legacy mode.") + yield from legacy_runner(request) + + +def legacy_runner(params): + """Runs the localnet.sh script in a subprocess and waits for it to start.""" # Get the environment variable for the script path script_path = os.getenv("LOCALNET_SH_PATH") @@ -31,41 +89,11 @@ def local_chain(request): pytest.skip("LOCALNET_SH_PATH environment variable is not set.") # Check if param is None, and handle it accordingly - args = "" if param is None else f"{param}" + args = "" if params is None else f"{params}" # Compile commands to send to process cmds = shlex.split(f"{script_path} {args}") - # Pattern match indicates node is compiled and ready - pattern = re.compile(r"Imported #1") - timestamp = int(time.time()) - - def wait_for_node_start(process, pattern): - while True: - line = process.stdout.readline() - if not line: - break - - print(line.strip()) - # 10 min as timeout - if int(time.time()) - timestamp > 20 * 60: - print("Subtensor not started in time") - raise TimeoutError - if pattern.search(line): - print("Node started!") - break - - # Start a background reader after pattern is found - # To prevent the buffer filling up - def read_output(): - while True: - line = process.stdout.readline() - if not line: - break - - reader_thread = threading.Thread(target=read_output, daemon=True) - reader_thread.start() - with subprocess.Popen( cmds, start_new_session=True, @@ -74,11 +102,12 @@ def read_output(): text=True, ) as process: try: - wait_for_node_start(process, pattern) + wait_for_node_start(process) except TimeoutError: raise else: - yield SubstrateInterface(url="ws://127.0.0.1:9944") + with SubstrateInterface(url="ws://127.0.0.1:9944") as substrate: + yield substrate finally: # Terminate the process group (includes all child processes) os.killpg(os.getpgid(process.pid), signal.SIGTERM) @@ -91,6 +120,100 @@ def read_output(): process.wait() +def docker_runner(params): + """Starts a Docker container before tests and gracefully terminates it after.""" + + def is_docker_running(): + """Check if Docker has been run.""" + try: + subprocess.run( + ["docker", "info"], + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, + check=True, + ) + return True + except subprocess.CalledProcessError: + return False + + def try_start_docker(): + """Run docker based on OS.""" + try: + subprocess.run(["open", "-a", "Docker"], check=True) # macOS + except (FileNotFoundError, subprocess.CalledProcessError): + try: + subprocess.run(["systemctl", "start", "docker"], check=True) # Linux + except (FileNotFoundError, subprocess.CalledProcessError): + try: + subprocess.run( + ["sudo", "service", "docker", "start"], check=True + ) # Linux alternative + except (FileNotFoundError, subprocess.CalledProcessError): + print("Failed to start Docker. Manual start may be required.") + return False + + # Wait Docker run 10 attempts with 3 sec waits + for _ in range(10): + if is_docker_running(): + return True + time.sleep(3) + + print("Docker wasn't run. Manual start may be required.") + return False + + container_name = f"test_local_chain_{str(time.time()).replace(".", "_")}" + image_name = "ghcr.io/opentensor/subtensor-localnet:latest" + + # Command to start container + cmds = [ + "docker", + "run", + "--rm", + "--name", + container_name, + "-p", + "9944:9944", + "-p", + "9945:9945", + image_name, + params, + ] + + try_start_docker() + + # Start container + with subprocess.Popen( + cmds, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + start_new_session=True, + ) as process: + try: + try: + wait_for_node_start(process, timestamp=int(time.time())) + except TimeoutError: + raise + + result = subprocess.run( + ["docker", "ps", "-q", "-f", f"name={container_name}"], + capture_output=True, + text=True, + ) + if not result.stdout.strip(): + raise RuntimeError("Docker container failed to start.") + + with SubstrateInterface(url="ws://127.0.0.1:9944") as substrate: + yield substrate + + finally: + try: + subprocess.run(["docker", "kill", container_name]) + process.wait() + except subprocess.TimeoutExpired: + os.killpg(os.getpgid(process.pid), signal.SIGKILL) + + @pytest.fixture(scope="session") def templates(): with Templates() as templates: