Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions .gitignore
Original file line number Diff line number Diff line change
Expand Up @@ -71,3 +71,4 @@ CLAUDE.MD
/assets/teleop_certs/

/.mcp.json
*.speedscope.json
8 changes: 6 additions & 2 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -204,7 +204,10 @@ See below a simple robot connection module that sends streams of continuous `cmd

```py
import threading, time, numpy as np
from dimos.core import In, Module, Out, rpc, autoconnect
from dimos.core.blueprints import autoconnect
from dimos.core.core import rpc
from dimos.core.module import Module
from dimos.core.stream import In, Out
from dimos.msgs.geometry_msgs import Twist
from dimos.msgs.sensor_msgs import Image, ImageFormat

Expand Down Expand Up @@ -249,7 +252,8 @@ Blueprints can be composed, remapped, and have transports overridden if `autocon

A blueprint example that connects the image stream from a robot to an LLM Agent for reasoning and action execution.
```py
from dimos.core import autoconnect, LCMTransport
from dimos.core.blueprints import autoconnect
from dimos.core.transport import LCMTransport
from dimos.msgs.sensor_msgs import Image
from dimos.robot.unitree.go2.connection import go2_connection
from dimos.agents.agent import agent
Expand Down
4 changes: 3 additions & 1 deletion dimos/agents/agent.py
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,6 @@
from typing import TYPE_CHECKING, Any, Protocol
import uuid

from langchain.agents import create_agent
from langchain_core.messages import HumanMessage
from langchain_core.messages.base import BaseMessage
from langchain_core.tools import StructuredTool
Expand Down Expand Up @@ -104,6 +103,9 @@ def on_system_modules(self, modules: list[RPCClient]) -> None:
model = MockModel(json_path=self.config.model_fixture)

with self._lock:
# Here to prevent unwanted imports in the file.
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

@paul-nechifor I assume LLM generated comments its on every import shift. not useful and looks ugly

Copy link
Contributor Author

@paul-nechifor paul-nechifor Feb 28, 2026

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

They're not LLM generated. I specifically added them there to indicate that those imports should not be moved to the top because they cause imports (like that of torch) which slow down the start of dimos run.

Without those comments, people might move them to the top. I often move unnecessary local imports because they're added by LLMs. LLMs often prefer to add local imports because they have the habit of producing the smallest diff possible, even when it doesn't make sense.

from langchain.agents import create_agent

self._state_graph = create_agent(
model=model,
tools=_get_tools_from_modules(self, modules, self.rpc),
Expand Down
6 changes: 1 addition & 5 deletions dimos/agents/conftest.py
Original file line number Diff line number Diff line change
Expand Up @@ -42,7 +42,6 @@ def fn(
*,
blueprints,
messages: list[BaseMessage],
dask: bool = False,
system_prompt: str | None = None,
fixture: str | None = None,
) -> list[BaseMessage]:
Expand Down Expand Up @@ -79,10 +78,7 @@ def on_message(msg: BaseMessage) -> None:
AgentTestRunner.blueprint(messages=messages),
)

global_config.update(
viewer_backend="none",
dask=dask,
)
global_config.update(viewer_backend="none")

nonlocal coordinator
coordinator = blueprint.build()
Expand Down
15 changes: 10 additions & 5 deletions dimos/agents/demo_agent.py
Original file line number Diff line number Diff line change
Expand Up @@ -20,13 +20,18 @@

demo_agent = autoconnect(Agent.blueprint())


def _create_webcam() -> Webcam:
return Webcam(
camera_index=0,
fps=15,
camera_info=zed.CameraInfo.SingleWebcam,
)


demo_agent_camera = autoconnect(
Agent.blueprint(),
camera_module(
hardware=lambda: Webcam(
camera_index=0,
fps=15,
camera_info=zed.CameraInfo.SingleWebcam,
),
hardware=_create_webcam,
),
)
6 changes: 1 addition & 5 deletions dimos/agents/mcp/conftest.py
Original file line number Diff line number Diff line change
Expand Up @@ -43,7 +43,6 @@ def fn(
*,
blueprints,
messages: list[BaseMessage],
dask: bool = False,
system_prompt: str | None = None,
fixture: str | None = None,
) -> list[BaseMessage]:
Expand Down Expand Up @@ -78,10 +77,7 @@ def on_message(msg: BaseMessage) -> None:
AgentTestRunner.blueprint(messages=messages),
)

global_config.update(
viewer_backend="none",
dask=dask,
)
global_config.update(viewer_backend="none")

nonlocal coordinator
coordinator = blueprint.build()
Expand Down
7 changes: 4 additions & 3 deletions dimos/agents/mcp/mcp_server.py
Original file line number Diff line number Diff line change
Expand Up @@ -28,11 +28,12 @@
logger = setup_logger()


from dimos.core import Module, rpc # noqa: I001
from dimos.core.rpc_client import RpcCall, RPCClient

from starlette.requests import Request # noqa: TC002

from dimos.core.core import rpc
from dimos.core.module import Module
from dimos.core.rpc_client import RpcCall, RPCClient

if TYPE_CHECKING:
import concurrent.futures

Expand Down
8 changes: 2 additions & 6 deletions dimos/agents/mcp/test_mcp_client.py
Original file line number Diff line number Diff line change
Expand Up @@ -30,12 +30,10 @@ def add(self, x: int, y: int) -> str:


@pytest.mark.slow
@pytest.mark.parametrize("dask", [False, True])
def test_can_call_tool(dask, agent_setup):
def test_can_call_tool(agent_setup):
history = agent_setup(
blueprints=[Adder.blueprint()],
messages=[HumanMessage("What is 33333 + 100? Use the tool.")],
dask=dask,
)

assert "33433" in history[-1].content
Expand Down Expand Up @@ -67,16 +65,14 @@ def register_user(self, name: str) -> str:


@pytest.mark.slow
@pytest.mark.parametrize("dask", [False, True])
def test_can_call_again_on_error(dask, agent_setup):
def test_can_call_again_on_error(agent_setup):
history = agent_setup(
blueprints=[UserRegistration.blueprint()],
messages=[
HumanMessage(
"Register a user named 'Paul'. If there are errors, just try again until you succeed."
)
],
dask=dask,
)

assert any(message.content == "User name registered successfully." for message in history)
Expand Down
7 changes: 5 additions & 2 deletions dimos/agents/skills/navigation.py
Original file line number Diff line number Diff line change
Expand Up @@ -21,8 +21,7 @@
from dimos.core.core import rpc
from dimos.core.module import Module
from dimos.core.stream import In
from dimos.models.qwen.video_query import BBox
from dimos.models.vl.qwen import QwenVlModel
from dimos.models.qwen.bbox import BBox
from dimos.msgs.geometry_msgs import PoseStamped, Quaternion, Vector3
from dimos.msgs.geometry_msgs.Vector3 import make_vector3
from dimos.msgs.sensor_msgs import Image
Expand Down Expand Up @@ -59,6 +58,10 @@ class NavigationSkillContainer(Module):
def __init__(self) -> None:
super().__init__()
self._skill_started = False

# Here to prevent unwanted imports in the file.
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

and here

from dimos.models.vl.qwen import QwenVlModel

self._vl_model = QwenVlModel()

@rpc
Expand Down
9 changes: 6 additions & 3 deletions dimos/agents/skills/person_follow.py
Original file line number Diff line number Diff line change
Expand Up @@ -26,8 +26,7 @@
from dimos.core.global_config import GlobalConfig
from dimos.core.module import Module
from dimos.core.stream import In, Out
from dimos.models.qwen.video_query import BBox
from dimos.models.segmentation.edge_tam import EdgeTAMProcessor
from dimos.models.qwen.bbox import BBox
from dimos.models.vl.qwen import QwenVlModel
from dimos.msgs.geometry_msgs import Twist
from dimos.msgs.sensor_msgs import CameraInfo, Image, PointCloud2
Expand All @@ -37,6 +36,7 @@
from dimos.utils.logging_config import setup_logger

if TYPE_CHECKING:
from dimos.models.segmentation.edge_tam import EdgeTAMProcessor
from dimos.models.vl.base import VlModel

logger = setup_logger()
Expand Down Expand Up @@ -176,6 +176,9 @@ def _follow_person(self, query: str, initial_bbox: BBox) -> str:

with self._lock:
if self._tracker is None:
# Here to prevent unwanted imports in the file.
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

and here

from dimos.models.segmentation.edge_tam import EdgeTAMProcessor

self._tracker = EdgeTAMProcessor()
tracker = self._tracker
latest_image = self._latest_image
Expand All @@ -202,7 +205,7 @@ def _follow_person(self, query: str, initial_bbox: BBox) -> str:
"the 'stop_following' tool."
)

def _follow_loop(self, tracker: EdgeTAMProcessor, query: str) -> None:
def _follow_loop(self, tracker: "EdgeTAMProcessor", query: str) -> None:
lost_count = 0
period = 1.0 / self._frequency
next_time = time.monotonic()
Expand Down
8 changes: 2 additions & 6 deletions dimos/agents/test_agent.py
Original file line number Diff line number Diff line change
Expand Up @@ -30,12 +30,10 @@ def add(self, x: int, y: int) -> str:


@pytest.mark.slow
@pytest.mark.parametrize("dask", [False, True])
def test_can_call_tool(dask, agent_setup):
def test_can_call_tool(agent_setup):
history = agent_setup(
blueprints=[Adder.blueprint()],
messages=[HumanMessage("What is 33333 + 100? Use the tool.")],
dask=dask,
)

assert "33433" in history[-1].content
Expand Down Expand Up @@ -69,16 +67,14 @@ def register_user(self, name: str) -> str:


@pytest.mark.slow
@pytest.mark.parametrize("dask", [False, True])
def test_can_call_again_on_error(dask, agent_setup):
def test_can_call_again_on_error(agent_setup):
history = agent_setup(
blueprints=[UserRegistration.blueprint()],
messages=[
HumanMessage(
"Register a user named 'Paul'. If there are errors, just try again until you succeed."
)
],
dask=dask,
)

assert any(message.content == "User name registered successfully." for message in history)
Expand Down
4 changes: 2 additions & 2 deletions dimos/agents/vlm_agent.py
Original file line number Diff line number Diff line change
Expand Up @@ -19,8 +19,8 @@
from langchain_core.messages import AIMessage, HumanMessage, SystemMessage

from dimos.agents.system_prompt import SYSTEM_PROMPT
from dimos.core import Module, rpc
from dimos.core.module import ModuleConfig
from dimos.core.core import rpc
from dimos.core.module import Module, ModuleConfig
from dimos.core.stream import In, Out
from dimos.msgs.sensor_msgs import Image
from dimos.utils.logging_config import setup_logger
Expand Down
3 changes: 2 additions & 1 deletion dimos/agents/vlm_stream_tester.py
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,8 @@

from langchain_core.messages import AIMessage, HumanMessage

from dimos.core import Module, rpc
from dimos.core.core import rpc
from dimos.core.module import Module
from dimos.core.stream import In, Out
from dimos.msgs.sensor_msgs import Image
from dimos.utils.logging_config import setup_logger
Expand Down
8 changes: 6 additions & 2 deletions dimos/agents/web_human_input.py
Original file line number Diff line number Diff line change
Expand Up @@ -18,10 +18,10 @@
import reactivex as rx
import reactivex.operators as ops

from dimos.core import Module, rpc
from dimos.core.core import rpc
from dimos.core.module import Module
from dimos.core.transport import pLCMTransport
from dimos.stream.audio.node_normalizer import AudioNormalizer
from dimos.stream.audio.stt.node_whisper import WhisperNode
from dimos.utils.logging_config import setup_logger
from dimos.web.robot_web_interface import RobotWebInterface

Expand Down Expand Up @@ -51,6 +51,10 @@ def start(self) -> None:
)

normalizer = AudioNormalizer()

# Here to prevent unwanted imports in the file.
from dimos.stream.audio.stt.node_whisper import WhisperNode

stt_node = WhisperNode()

# Connect audio pipeline: browser audio → normalizer → whisper
Expand Down
4 changes: 3 additions & 1 deletion dimos/agents_deprecated/memory/spatial_vector_db.py
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,6 @@

from typing import Any

import chromadb
import numpy as np

from dimos.agents_deprecated.memory.visual_memory import VisualMemory
Expand Down Expand Up @@ -57,6 +56,9 @@ def __init__( # type: ignore[no-untyped-def]
"""
self.collection_name = collection_name

# Here to prevent unwanted imports in the file.
import chromadb

# Use provided client or create in-memory client
self.client = chroma_client if chroma_client is not None else chromadb.Client()

Expand Down
4 changes: 3 additions & 1 deletion dimos/agents_deprecated/modules/base_agent.py
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,9 @@
from dimos.agents_deprecated.agent_message import AgentMessage
from dimos.agents_deprecated.agent_types import AgentResponse
from dimos.agents_deprecated.memory.base import AbstractAgentSemanticMemory
from dimos.core import In, Module, Out, rpc
from dimos.core.core import rpc
from dimos.core.module import Module
from dimos.core.stream import In, Out
from dimos.skills.skills import AbstractSkill, SkillLibrary
from dimos.utils.logging_config import setup_logger

Expand Down
6 changes: 3 additions & 3 deletions dimos/conftest.py
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,7 @@
from dotenv import load_dotenv
import pytest

from dimos.core.module_coordinator import ModuleCoordinator
from dimos.protocol.service.lcmservice import autoconf

load_dotenv()
Expand Down Expand Up @@ -86,9 +87,8 @@ def _autoconf(request):

@pytest.fixture(scope="module")
def dimos_cluster():
from dimos.core import start

dimos = start(4)
dimos = ModuleCoordinator()
dimos.start()
try:
yield dimos
finally:
Expand Down
17 changes: 8 additions & 9 deletions dimos/control/coordinator.py
Original file line number Diff line number Diff line change
Expand Up @@ -25,9 +25,8 @@
- Aggregated preemption notifications
"""

from __future__ import annotations

from dataclasses import dataclass, field
from pathlib import Path
import threading
import time
from typing import TYPE_CHECKING, Any
Expand All @@ -43,28 +42,28 @@
from dimos.control.hardware_interface import ConnectedHardware, ConnectedTwistBase
from dimos.control.task import ControlTask
from dimos.control.tick_loop import TickLoop
from dimos.core import In, Module, Out, rpc
from dimos.core.module import ModuleConfig
from dimos.core.core import rpc
from dimos.core.module import Module, ModuleConfig
from dimos.core.stream import In, Out
from dimos.hardware.drive_trains.spec import (
TwistBaseAdapter,
)
from dimos.hardware.manipulators.spec import ManipulatorAdapter
from dimos.msgs.geometry_msgs import (
PoseStamped, # noqa: TC001 - needed at runtime for In[PoseStamped]
Twist, # noqa: TC001 - needed at runtime for In[Twist]
PoseStamped,
Twist,
)
from dimos.msgs.sensor_msgs import (
JointState,
)
from dimos.teleop.quest.quest_types import (
Buttons, # noqa: TC001 - needed at runtime for In[Buttons]
Buttons,
)
from dimos.utils.logging_config import setup_logger

if TYPE_CHECKING:
from collections.abc import Callable
from pathlib import Path

from dimos.hardware.manipulators.spec import ManipulatorAdapter

logger = setup_logger()

Expand Down
Loading
Loading