diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index f9afc96de5..5dc19917e5 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -205,6 +205,21 @@ jobs: cmd: "pytest -m lcm" dev-image: dev:${{ (needs.check-changes.outputs.python == 'true' || needs.check-changes.outputs.dev == 'true') && needs.dev.result == 'success' && needs.check-changes.outputs.branch-tag || 'dev' }} + run-integration-tests: + needs: [check-changes, dev] + if: always() + uses: ./.github/workflows/tests.yml + secrets: inherit + with: + should-run: ${{ + needs.check-changes.result == 'success' && + ((needs.dev.result == 'success') || + (needs.dev.result == 'skipped' && + needs.check-changes.outputs.tests == 'true')) + }} + cmd: "pytest -m integration" + dev-image: dev:${{ (needs.check-changes.outputs.python == 'true' || needs.check-changes.outputs.dev == 'true') && needs.dev.result == 'success' && needs.check-changes.outputs.branch-tag || 'dev' }} + run-mypy: needs: [check-changes, ros-dev] if: always() diff --git a/bin/pytest-slow b/bin/pytest-slow new file mode 100755 index 0000000000..702445ec85 --- /dev/null +++ b/bin/pytest-slow @@ -0,0 +1,6 @@ +#!/usr/bin/env bash + +set -euo pipefail + +. .venv/bin/activate +exec pytest "$@" -m 'not (tool or cuda or gpu or module or temporal)' dimos diff --git a/dimos/agents/skills/test_navigation.py b/dimos/agents/skills/test_navigation.py index 588b55a602..67e0429cb5 100644 --- a/dimos/agents/skills/test_navigation.py +++ b/dimos/agents/skills/test_navigation.py @@ -12,12 +12,12 @@ # See the License for the specific language governing permissions and # limitations under the License. +import pytest from dimos.msgs.geometry_msgs import PoseStamped, Vector3 from dimos.utils.transform_utils import euler_to_quaternion -# @pytest.mark.skip def test_stop_movement(create_navigation_agent, navigation_skill_container, mocker) -> None: cancel_goal_mock = mocker.Mock() stop_exploration_mock = mocker.Mock() @@ -35,6 +35,7 @@ def test_stop_movement(create_navigation_agent, navigation_skill_container, mock stop_exploration_mock.assert_called_once_with() +@pytest.mark.integration def test_take_a_look_around(create_navigation_agent, navigation_skill_container, mocker) -> None: explore_mock = mocker.Mock() is_exploration_active_mock = mocker.Mock() diff --git a/dimos/agents/test_agent_fake.py b/dimos/agents/test_agent_fake.py index 367985a356..e544765758 100644 --- a/dimos/agents/test_agent_fake.py +++ b/dimos/agents/test_agent_fake.py @@ -12,13 +12,17 @@ # See the License for the specific language governing permissions and # limitations under the License. +import pytest + +@pytest.mark.integration def test_what_is_your_name(create_potato_agent) -> None: agent = create_potato_agent(fixture="test_what_is_your_name.json") response = agent.query("hi there, please tell me what's your name?") assert "Mr. Potato" in response +@pytest.mark.integration def test_how_much_is_124181112_plus_124124(create_potato_agent) -> None: agent = create_potato_agent(fixture="test_how_much_is_124181112_plus_124124.json") @@ -29,6 +33,7 @@ def test_how_much_is_124181112_plus_124124(create_potato_agent) -> None: assert "999000000" in response.replace(",", "") +@pytest.mark.integration def test_what_do_you_see_in_this_picture(create_potato_agent) -> None: agent = create_potato_agent(fixture="test_what_do_you_see_in_this_picture.json") diff --git a/dimos/agents/test_mock_agent.py b/dimos/agents/test_mock_agent.py index c711e23143..6cd584209e 100644 --- a/dimos/agents/test_mock_agent.py +++ b/dimos/agents/test_mock_agent.py @@ -30,6 +30,7 @@ from dimos.robot.unitree_webrtc.type.lidar import LidarMessage +@pytest.mark.integration def test_tool_call() -> None: """Test agent initialization and tool call execution.""" # Create a fake model that will respond with tool calls @@ -74,6 +75,7 @@ def test_tool_call() -> None: agent.stop() +@pytest.mark.integration def test_image_tool_call() -> None: """Test agent with image tool call execution.""" dimos = start(2) diff --git a/dimos/control/test_control.py b/dimos/control/test_control.py index e35a9853d5..2522affa60 100644 --- a/dimos/control/test_control.py +++ b/dimos/control/test_control.py @@ -36,6 +36,7 @@ TrajectoryState, ) from dimos.control.tick_loop import TickLoop +from dimos.hardware.manipulators.spec import ManipulatorBackend from dimos.msgs.trajectory_msgs import JointTrajectory, TrajectoryPoint # ============================================================================= @@ -46,9 +47,8 @@ @pytest.fixture def mock_backend(): """Create a mock manipulator backend.""" - backend = MagicMock() + backend = MagicMock(spec=ManipulatorBackend) backend.get_dof.return_value = 6 - backend.get_joint_names.return_value = [f"joint{i + 1}" for i in range(6)] backend.read_joint_positions.return_value = [0.0] * 6 backend.read_joint_velocities.return_value = [0.0] * 6 backend.read_joint_efforts.return_value = [0.0] * 6 diff --git a/dimos/core/test_blueprints.py b/dimos/core/test_blueprints.py index 7a99a23abe..a8b9354f70 100644 --- a/dimos/core/test_blueprints.py +++ b/dimos/core/test_blueprints.py @@ -162,6 +162,7 @@ def test_global_config() -> None: assert blueprint_set.global_config_overrides["option2"] == 42 +@pytest.mark.integration def test_build_happy_path() -> None: pubsub.lcm.autoconf() @@ -272,6 +273,7 @@ class Module3(Module): blueprint_set_remapped._verify_no_name_conflicts() +@pytest.mark.integration def test_remapping() -> None: """Test that remapping connections works correctly.""" pubsub.lcm.autoconf() @@ -351,6 +353,7 @@ def test_future_annotations_support() -> None: ) +@pytest.mark.integration def test_future_annotations_autoconnect() -> None: """Test that autoconnect works with modules using `from __future__ import annotations`.""" diff --git a/dimos/e2e_tests/test_control_orchestrator.py b/dimos/e2e_tests/test_control_orchestrator.py index a97c045ce6..967a219ceb 100644 --- a/dimos/e2e_tests/test_control_orchestrator.py +++ b/dimos/e2e_tests/test_control_orchestrator.py @@ -33,6 +33,7 @@ @pytest.mark.skipif(bool(os.getenv("CI")), reason="LCM doesn't work in CI.") +@pytest.mark.e2e class TestControlOrchestratorE2E: """End-to-end tests for ControlOrchestrator.""" diff --git a/dimos/e2e_tests/test_dimos_cli_e2e.py b/dimos/e2e_tests/test_dimos_cli_e2e.py index 7571e113ad..c381d35cfa 100644 --- a/dimos/e2e_tests/test_dimos_cli_e2e.py +++ b/dimos/e2e_tests/test_dimos_cli_e2e.py @@ -19,6 +19,7 @@ @pytest.mark.skipif(bool(os.getenv("CI")), reason="LCM spy doesn't work in CI.") @pytest.mark.skipif(not os.getenv("OPENAI_API_KEY"), reason="OPENAI_API_KEY not set.") +@pytest.mark.e2e def test_dimos_skills(lcm_spy, start_blueprint, human_input) -> None: lcm_spy.save_topic("/rpc/DemoCalculatorSkill/set_AgentSpec_register_skills/res") lcm_spy.save_topic("/rpc/HumanInput/start/res") diff --git a/dimos/mapping/occupancy/test_extrude_occupancy.py b/dimos/mapping/occupancy/test_extrude_occupancy.py index 81caba7c8d..88f05d7780 100644 --- a/dimos/mapping/occupancy/test_extrude_occupancy.py +++ b/dimos/mapping/occupancy/test_extrude_occupancy.py @@ -12,10 +12,13 @@ # See the License for the specific language governing permissions and # limitations under the License. +import pytest + from dimos.mapping.occupancy.extrude_occupancy import generate_mujoco_scene from dimos.utils.data import get_data +@pytest.mark.integration def test_generate_mujoco_scene(occupancy) -> None: with open(get_data("expected_occupancy_scene.xml")) as f: expected = f.read() diff --git a/dimos/msgs/sensor_msgs/image_impls/test_image_backends.py b/dimos/msgs/sensor_msgs/image_impls/test_image_backends.py index b1de0ac777..d3a5bf91e2 100644 --- a/dimos/msgs/sensor_msgs/image_impls/test_image_backends.py +++ b/dimos/msgs/sensor_msgs/image_impls/test_image_backends.py @@ -356,6 +356,7 @@ def test_perf_resize(alloc_timer) -> None: print(f"resize (avg per call) cpu={cpu_t:.6f}s") +@pytest.mark.integration def test_perf_sharpness(alloc_timer) -> None: """Test sharpness performance with NumpyImage always, add CudaImage when available.""" arr = _prepare_image(ImageFormat.BGR, (480, 640, 3)) diff --git a/dimos/perception/experimental/temporal_memory/test_temporal_memory_module.py b/dimos/perception/experimental/temporal_memory/test_temporal_memory_module.py index b4af1b4020..8029bd6454 100644 --- a/dimos/perception/experimental/temporal_memory/test_temporal_memory_module.py +++ b/dimos/perception/experimental/temporal_memory/test_temporal_memory_module.py @@ -81,6 +81,7 @@ def stop(self) -> None: @pytest.mark.skipif(bool(os.getenv("CI")), reason="LCM replay + dataset not CI-safe.") @pytest.mark.skipif(not os.getenv("OPENAI_API_KEY"), reason="OPENAI_API_KEY not set.") +@pytest.mark.temporal # TODO: This never finishes for me. class TestTemporalMemoryModule: @pytest.fixture(scope="function") def temp_dir(self): diff --git a/dimos/protocol/mcp/test_mcp_module.py b/dimos/protocol/mcp/test_mcp_module.py index 68d6a137f8..2a247e6ff0 100644 --- a/dimos/protocol/mcp/test_mcp_module.py +++ b/dimos/protocol/mcp/test_mcp_module.py @@ -133,6 +133,7 @@ async def wait_for_updates(self) -> bool: assert "Error:" in response["result"]["content"][0]["text"] +@pytest.mark.integration def test_mcp_end_to_end_lcm_bridge() -> None: try: import lcm # type: ignore[import-untyped] diff --git a/dimos/protocol/rpc/test_spec.py b/dimos/protocol/rpc/test_spec.py index 9fb8f65eb7..c29db13703 100644 --- a/dimos/protocol/rpc/test_spec.py +++ b/dimos/protocol/rpc/test_spec.py @@ -293,6 +293,7 @@ def callback(val) -> None: unsub_server() +@pytest.mark.integration @pytest.mark.parametrize("rpc_context, impl_name", testdata) def test_timeout(rpc_context, impl_name: str) -> None: """Test that RPC calls properly timeout.""" diff --git a/dimos/protocol/skill/test_coordinator.py b/dimos/protocol/skill/test_coordinator.py index acaad98dda..bd00ea69c2 100644 --- a/dimos/protocol/skill/test_coordinator.py +++ b/dimos/protocol/skill/test_coordinator.py @@ -96,6 +96,7 @@ def take_photo(self) -> Image: return img +@pytest.mark.integration @pytest.mark.asyncio # type: ignore[untyped-decorator] async def test_coordinator_parallel_calls() -> None: container = SkillContainerTest() @@ -136,6 +137,7 @@ async def test_coordinator_parallel_calls() -> None: skillCoordinator.stop() +@pytest.mark.integration @pytest.mark.asyncio # type: ignore[untyped-decorator] async def test_coordinator_generator() -> None: container = SkillContainerTest() diff --git a/dimos/robot/drone/test_drone.py b/dimos/robot/drone/test_drone.py index bfbaa9ed54..6433f77ec5 100644 --- a/dimos/robot/drone/test_drone.py +++ b/dimos/robot/drone/test_drone.py @@ -24,6 +24,7 @@ from unittest.mock import MagicMock, patch import numpy as np +import pytest from dimos.msgs.geometry_msgs import PoseStamped, Quaternion, Vector3 from dimos.msgs.sensor_msgs import Image, ImageFormat diff --git a/dimos/robot/test_all_blueprints.py b/dimos/robot/test_all_blueprints.py index 684cdd938a..7e5fa6970c 100644 --- a/dimos/robot/test_all_blueprints.py +++ b/dimos/robot/test_all_blueprints.py @@ -24,6 +24,7 @@ } +@pytest.mark.integration @pytest.mark.parametrize("blueprint_name", all_blueprints.keys()) def test_all_blueprints_are_valid(blueprint_name: str) -> None: """Test that all blueprints in all_blueprints are valid ModuleBlueprintSet instances.""" diff --git a/dimos/types/test_weaklist.py b/dimos/types/test_weaklist.py index 990cc0d164..06f9f851ce 100644 --- a/dimos/types/test_weaklist.py +++ b/dimos/types/test_weaklist.py @@ -54,6 +54,7 @@ def test_weaklist_basic_operations() -> None: assert SampleObject(4) not in wl +@pytest.mark.integration def test_weaklist_auto_removal() -> None: """Test that objects are automatically removed when garbage collected.""" wl = WeakList() @@ -136,6 +137,7 @@ def test_weaklist_clear() -> None: assert obj1 not in wl +@pytest.mark.integration def test_weaklist_iteration_during_modification() -> None: """Test that iteration works even if objects are deleted during iteration.""" wl = WeakList() diff --git a/dimos/utils/test_reactive.py b/dimos/utils/test_reactive.py index a0f3fe42ef..5bfc0a590f 100644 --- a/dimos/utils/test_reactive.py +++ b/dimos/utils/test_reactive.py @@ -82,6 +82,7 @@ def _dispose() -> None: return proxy +@pytest.mark.integration def test_backpressure_handling() -> None: # Create a dedicated scheduler for this test to avoid thread leaks test_scheduler = ThreadPoolScheduler(max_workers=8) @@ -141,6 +142,7 @@ def test_backpressure_handling() -> None: test_scheduler.executor.shutdown(wait=True) +@pytest.mark.integration def test_getter_streaming_blocking() -> None: source = dispose_spy( rx.interval(0.2).pipe(ops.map(lambda i: np.array([i, i + 1, i + 2])), ops.take(50)) @@ -175,6 +177,7 @@ def test_getter_streaming_blocking_timeout() -> None: assert source.is_disposed() +@pytest.mark.integration def test_getter_streaming_nonblocking() -> None: source = dispose_spy(rx.interval(0.2).pipe(ops.take(50))) diff --git a/pyproject.toml b/pyproject.toml index 65e586b7ac..f4fc64af1f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -349,12 +349,15 @@ markers = [ "lcm: tests that run actual LCM bus (can't execute in CI)", "module: tests that need to run directly as modules", "gpu: tests that require GPU", - "tofix: temporarily disabled test" + "tofix: temporarily disabled test", + "e2e: end to end tests", + "temporal: not working for me", + "integration: slower integration tests", ] env = [ "GOOGLE_MAPS_API_KEY=AIzafake_google_key" ] -addopts = "-v -p no:warnings -ra --color=yes -m 'not vis and not benchmark and not exclude and not tool and not needsdata and not lcm and not ros and not heavy and not gpu and not module and not tofix'" +addopts = "-v -p no:warnings -ra --color=yes -m 'not (vis or benchmark or exclude or tool or needsdata or lcm or ros or heavy or gpu or module or tofix or e2e or temporal or integration)'" asyncio_mode = "auto" asyncio_default_fixture_loop_scope = "function"