From c09a175605f7764eb22da69e4b2e3fb85cb0a1ca Mon Sep 17 00:00:00 2001 From: Paul Nechifor Date: Thu, 13 Nov 2025 14:26:20 +0200 Subject: [PATCH 1/4] local agents --- dimos/agents2/agent.py | 46 +++++++++++++++++-- dimos/robot/all_blueprints.py | 2 + dimos/robot/cli/test_dimos_robot_e2e.py | 4 +- .../unitree_webrtc/unitree_go2_blueprints.py | 29 ++++++++++-- dimos/utils/cli/human/humanclianim.py | 9 ++-- pyproject.toml | 3 ++ 6 files changed, 79 insertions(+), 14 deletions(-) diff --git a/dimos/agents2/agent.py b/dimos/agents2/agent.py index e58e1aa9a3..830d4a6402 100644 --- a/dimos/agents2/agent.py +++ b/dimos/agents2/agent.py @@ -27,6 +27,8 @@ ToolCall, ToolMessage, ) +from langchain_huggingface import ChatHuggingFace, HuggingFacePipeline +import ollama from dimos.agents2.spec import AgentSpec, Model, Provider from dimos.agents2.system_prompt import get_system_prompt @@ -159,6 +161,14 @@ def snapshot_to_messages( } +def _ensure_ollama_model(model_name: str) -> None: + available_models = ollama.list() + model_exists = any(model_name == m.model for m in available_models.models) + if not model_exists: + logger.info(f"Ollama model '{model_name}' not found. Pulling...") + ollama.pull(model_name) + + # Agent class job is to glue skill coordinator state to an agent, builds langchain messages class Agent(AgentSpec): system_message: SystemMessage @@ -192,9 +202,25 @@ def __init__( if self.config.model_instance: self._llm = self.config.model_instance else: - self._llm = init_chat_model( - model_provider=self.config.provider, model=self.config.model - ) + # For Ollama provider, ensure the model is available before initializing + if self.config.provider.value.lower() == "ollama": + _ensure_ollama_model(self.config.model) + + # For HuggingFace, we need to create a pipeline and wrap it in ChatHuggingFace + if self.config.provider.value.lower() == "huggingface": + llm = HuggingFacePipeline.from_model_id( + model_id=self.config.model, + task="text-generation", + pipeline_kwargs={ + "max_new_tokens": 512, + "temperature": 0.7, + }, + ) + self._llm = ChatHuggingFace(llm=llm, model_id=self.config.model) + else: + self._llm = init_chat_model( + model_provider=self.config.provider, model=self.config.model + ) @rpc def get_agent_id(self) -> str: @@ -278,7 +304,19 @@ def _get_state() -> str: # history() builds our message history dynamically # ensures we include latest system state, but not old ones. - msg = self._llm.invoke(self.history()) + messages = self.history() + + # Some LLMs don't work without any human messages. Add an initial one. + if len(messages) == 1 and isinstance(messages[0], SystemMessage): + messages.append( + HumanMessage( + "Everything is initialized. I'll let you know when you should act." + ) + ) + self.append_history(messages[-1]) + + msg = self._llm.invoke(messages) + self.append_history(msg) logger.info(f"Agent response: {msg.content}") diff --git a/dimos/robot/all_blueprints.py b/dimos/robot/all_blueprints.py index 1054b8133c..d345bc35cf 100644 --- a/dimos/robot/all_blueprints.py +++ b/dimos/robot/all_blueprints.py @@ -22,6 +22,8 @@ "unitree-go2-jpegshm": "dimos.robot.unitree_webrtc.unitree_go2_blueprints:standard_with_jpegshm", "unitree-go2-jpeglcm": "dimos.robot.unitree_webrtc.unitree_go2_blueprints:standard_with_jpeglcm", "unitree-go2-agentic": "dimos.robot.unitree_webrtc.unitree_go2_blueprints:agentic", + "unitree-go2-agentic-ollama": "dimos.robot.unitree_webrtc.unitree_go2_blueprints:agentic_ollama", + "unitree-go2-agentic-huggingface": "dimos.robot.unitree_webrtc.unitree_go2_blueprints:agentic_huggingface", "unitree-g1": "dimos.robot.unitree_webrtc.unitree_g1_blueprints:standard", "unitree-g1-bt-nav": "dimos.robot.unitree_webrtc.unitree_g1_blueprints:standard_bt_nav", "unitree-g1-basic": "dimos.robot.unitree_webrtc.unitree_g1_blueprints:basic_ros", diff --git a/dimos/robot/cli/test_dimos_robot_e2e.py b/dimos/robot/cli/test_dimos_robot_e2e.py index 8ae93b4814..72cf949ee6 100644 --- a/dimos/robot/cli/test_dimos_robot_e2e.py +++ b/dimos/robot/cli/test_dimos_robot_e2e.py @@ -69,7 +69,7 @@ class DimosRobotCall: def __init__(self) -> None: self.process = None - def start(self): + def start(self) -> None: self.process = subprocess.Popen( ["dimos", "run", "demo-skill"], stdout=subprocess.PIPE, @@ -143,7 +143,7 @@ def send_human_input(message: str) -> None: @pytest.mark.skipif(bool(os.getenv("CI")), reason="LCM spy doesn't work in CI.") -def test_dimos_robot_demo_e2e(lcm_spy, dimos_robot_call, human_input): +def test_dimos_robot_demo_e2e(lcm_spy, dimos_robot_call, human_input) -> None: lcm_spy.wait_for_topic("/rpc/DemoCalculatorSkill/set_LlmAgent_register_skills/res") lcm_spy.wait_for_topic("/rpc/HumanInput/start/res") lcm_spy.wait_for_message_content("/agent", b"AIMessage") diff --git a/dimos/robot/unitree_webrtc/unitree_go2_blueprints.py b/dimos/robot/unitree_webrtc/unitree_go2_blueprints.py index 8973f6cd68..d9fbdfb62e 100644 --- a/dimos/robot/unitree_webrtc/unitree_go2_blueprints.py +++ b/dimos/robot/unitree_webrtc/unitree_go2_blueprints.py @@ -19,6 +19,7 @@ from dimos.agents2.agent import llm_agent from dimos.agents2.cli.human import human_input from dimos.agents2.skills.navigation import navigation_skill +from dimos.agents2.spec import Provider from dimos.constants import DEFAULT_CAPACITY_COLOR_IMAGE from dimos.core.blueprints import autoconnect from dimos.core.transport import JpegLcmTransport, JpegShmTransport, LCMTransport, pSHMTransport @@ -107,10 +108,32 @@ ), ) -agentic = autoconnect( - standard, - llm_agent(), +_common_agentic = autoconnect( human_input(), navigation_skill(), unitree_skills(), ) + +agentic = autoconnect( + standard, + llm_agent(), + _common_agentic, +) + +agentic_ollama = autoconnect( + standard, + llm_agent( + model="qwen3:8b", + provider=Provider.OLLAMA, + ), + _common_agentic, +) + +agentic_huggingface = autoconnect( + standard, + llm_agent( + model="Qwen/Qwen2.5-1.5B-Instruct", + provider=Provider.HUGGINGFACE, + ), + _common_agentic, +) diff --git a/dimos/utils/cli/human/humanclianim.py b/dimos/utils/cli/human/humanclianim.py index a0349eedf8..8b6aae059e 100644 --- a/dimos/utils/cli/human/humanclianim.py +++ b/dimos/utils/cli/human/humanclianim.py @@ -30,7 +30,7 @@ print(theme.ACCENT) -def import_cli_in_background(): +def import_cli_in_background() -> None: """Import the heavy CLI modules in the background""" global _humancli_main try: @@ -43,7 +43,7 @@ def import_cli_in_background(): _import_complete.set() -def get_effect_config(effect_name): +def get_effect_config(effect_name: str): """Get hardcoded configuration for a specific effect""" # Hardcoded configs for each effect global_config = { @@ -79,7 +79,7 @@ def get_effect_config(effect_name): return {**configs.get(effect_name, {}), **global_config} -def run_banner_animation(): +def run_banner_animation() -> None: """Run the ASCII banner animation before launching Textual""" # Check if we should animate @@ -90,7 +90,6 @@ def run_banner_animation(): return # Skip animation from terminaltexteffects.effects.effect_beams import Beams from terminaltexteffects.effects.effect_burn import Burn - from terminaltexteffects.effects.effect_colorshift import ColorShift from terminaltexteffects.effects.effect_decrypt import Decrypt from terminaltexteffects.effects.effect_expand import Expand from terminaltexteffects.effects.effect_highlight import Highlight @@ -151,7 +150,7 @@ def run_banner_animation(): print("\033[2J\033[H", end="") -def main(): +def main() -> None: """Main entry point - run animation then launch the real CLI""" # Start importing CLI in background (this is slow) diff --git a/pyproject.toml b/pyproject.toml index 1631baed36..e1967603e7 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -57,6 +57,9 @@ dependencies = [ "langchain-core>=0.3.72", "langchain-openai>=0.3.28", "langchain-text-splitters>=0.3.9", + "langchain-huggingface>=0.3.1", + "langchain-ollama>=0.3.10", + "bitsandbytes>=0.48.2,<1.0", # Class Extraction "pydantic", From 1a8b4774215e318d7527d5c851f491cf18727179 Mon Sep 17 00:00:00 2001 From: Paul Nechifor Date: Tue, 18 Nov 2025 05:10:00 +0200 Subject: [PATCH 2/4] error if ollama is not installed. --- dimos/agents2/agent.py | 12 +----- dimos/agents2/ollama_agent.py | 39 +++++++++++++++++++ dimos/core/blueprints.py | 35 ++++++++++++++++- .../unitree_webrtc/unitree_go2_blueprints.py | 3 ++ 4 files changed, 78 insertions(+), 11 deletions(-) create mode 100644 dimos/agents2/ollama_agent.py diff --git a/dimos/agents2/agent.py b/dimos/agents2/agent.py index 830d4a6402..5b46cc8f33 100644 --- a/dimos/agents2/agent.py +++ b/dimos/agents2/agent.py @@ -28,8 +28,8 @@ ToolMessage, ) from langchain_huggingface import ChatHuggingFace, HuggingFacePipeline -import ollama +from dimos.agents2.ollama_agent import ensure_ollama_model from dimos.agents2.spec import AgentSpec, Model, Provider from dimos.agents2.system_prompt import get_system_prompt from dimos.core import DimosCluster, rpc @@ -161,14 +161,6 @@ def snapshot_to_messages( } -def _ensure_ollama_model(model_name: str) -> None: - available_models = ollama.list() - model_exists = any(model_name == m.model for m in available_models.models) - if not model_exists: - logger.info(f"Ollama model '{model_name}' not found. Pulling...") - ollama.pull(model_name) - - # Agent class job is to glue skill coordinator state to an agent, builds langchain messages class Agent(AgentSpec): system_message: SystemMessage @@ -204,7 +196,7 @@ def __init__( else: # For Ollama provider, ensure the model is available before initializing if self.config.provider.value.lower() == "ollama": - _ensure_ollama_model(self.config.model) + ensure_ollama_model(self.config.model) # For HuggingFace, we need to create a pipeline and wrap it in ChatHuggingFace if self.config.provider.value.lower() == "huggingface": diff --git a/dimos/agents2/ollama_agent.py b/dimos/agents2/ollama_agent.py new file mode 100644 index 0000000000..26179d7418 --- /dev/null +++ b/dimos/agents2/ollama_agent.py @@ -0,0 +1,39 @@ +# Copyright 2025 Dimensional Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import ollama + +from dimos.utils.logging_config import setup_logger + +logger = setup_logger(__file__) + + +def ensure_ollama_model(model_name: str) -> None: + available_models = ollama.list() + model_exists = any(model_name == m.model for m in available_models.models) + if not model_exists: + logger.info(f"Ollama model '{model_name}' not found. Pulling...") + ollama.pull(model_name) + + +def ollama_installed() -> str | None: + try: + ollama.list() + return None + except Exception: + return ( + "Cannot connect to Ollama daemon. Please ensure Ollama is installed and running.\n" + "\n" + " For installation instructions, visit https://ollama.com/download" + ) diff --git a/dimos/core/blueprints.py b/dimos/core/blueprints.py index 45aa617571..d46139cd6d 100644 --- a/dimos/core/blueprints.py +++ b/dimos/core/blueprints.py @@ -14,11 +14,12 @@ from abc import ABC from collections import defaultdict -from collections.abc import Mapping +from collections.abc import Callable, Mapping from dataclasses import dataclass, field from functools import cached_property, reduce import inspect import operator +import sys from types import MappingProxyType from typing import Any, Literal, get_args, get_origin @@ -56,6 +57,7 @@ class ModuleBlueprintSet: remapping_map: Mapping[tuple[type[Module], str], str] = field( default_factory=lambda: MappingProxyType({}) ) + requirement_checks: tuple[Callable[[], str | None], ...] = field(default_factory=tuple) def transports(self, transports: dict[tuple[str, type], Any]) -> "ModuleBlueprintSet": return ModuleBlueprintSet( @@ -63,6 +65,7 @@ def transports(self, transports: dict[tuple[str, type], Any]) -> "ModuleBlueprin transport_map=MappingProxyType({**self.transport_map, **transports}), global_config_overrides=self.global_config_overrides, remapping_map=self.remapping_map, + requirement_checks=self.requirement_checks, ) def global_config(self, **kwargs: Any) -> "ModuleBlueprintSet": @@ -71,6 +74,7 @@ def global_config(self, **kwargs: Any) -> "ModuleBlueprintSet": transport_map=self.transport_map, global_config_overrides=MappingProxyType({**self.global_config_overrides, **kwargs}), remapping_map=self.remapping_map, + requirement_checks=self.requirement_checks, ) def remappings(self, remappings: list[tuple[type[Module], str, str]]) -> "ModuleBlueprintSet": @@ -83,6 +87,16 @@ def remappings(self, remappings: list[tuple[type[Module], str, str]]) -> "Module transport_map=self.transport_map, global_config_overrides=self.global_config_overrides, remapping_map=MappingProxyType(remappings_dict), + requirement_checks=self.requirement_checks, + ) + + def requirements(self, *checks: Callable[[], str | None]) -> "ModuleBlueprintSet": + return ModuleBlueprintSet( + blueprints=self.blueprints, + transport_map=self.transport_map, + global_config_overrides=self.global_config_overrides, + remapping_map=self.remapping_map, + requirement_checks=self.requirement_checks + tuple(checks), ) def _get_transport_for(self, name: str, type: type) -> Any: @@ -110,6 +124,21 @@ def _all_name_types(self) -> set[tuple[str, type]]: def _is_name_unique(self, name: str) -> bool: return sum(1 for n, _ in self._all_name_types if n == name) == 1 + def _check_requirements(self) -> None: + errors = [] + red = "\033[31m" + reset = "\033[0m" + + for check in self.requirement_checks: + error = check() + if error: + errors.append(error) + + if errors: + for error in errors: + print(f"{red}Error: {error}{reset}", file=sys.stderr) + sys.exit(1) + def _deploy_all_modules( self, module_coordinator: ModuleCoordinator, global_config: GlobalConfig ) -> None: @@ -209,6 +238,8 @@ def build(self, global_config: GlobalConfig | None = None) -> ModuleCoordinator: global_config = GlobalConfig() global_config = global_config.model_copy(update=self.global_config_overrides) + self._check_requirements() + module_coordinator = ModuleCoordinator(global_config=global_config) module_coordinator.start() @@ -258,12 +289,14 @@ def autoconnect(*blueprints: ModuleBlueprintSet) -> ModuleBlueprintSet: all_remappings = dict( reduce(operator.iadd, [list(x.remapping_map.items()) for x in blueprints], []) ) + all_requirement_checks = tuple(check for bs in blueprints for check in bs.requirement_checks) return ModuleBlueprintSet( blueprints=all_blueprints, transport_map=MappingProxyType(all_transports), global_config_overrides=MappingProxyType(all_config_overrides), remapping_map=MappingProxyType(all_remappings), + requirement_checks=all_requirement_checks, ) diff --git a/dimos/robot/unitree_webrtc/unitree_go2_blueprints.py b/dimos/robot/unitree_webrtc/unitree_go2_blueprints.py index d9fbdfb62e..2dab0afd06 100644 --- a/dimos/robot/unitree_webrtc/unitree_go2_blueprints.py +++ b/dimos/robot/unitree_webrtc/unitree_go2_blueprints.py @@ -18,6 +18,7 @@ from dimos.agents2.agent import llm_agent from dimos.agents2.cli.human import human_input +from dimos.agents2.ollama_agent import ollama_installed from dimos.agents2.skills.navigation import navigation_skill from dimos.agents2.spec import Provider from dimos.constants import DEFAULT_CAPACITY_COLOR_IMAGE @@ -127,6 +128,8 @@ provider=Provider.OLLAMA, ), _common_agentic, +).requirements( + ollama_installed, ) agentic_huggingface = autoconnect( From ada735cc149328ea5681152f3b7366e3c657cb0f Mon Sep 17 00:00:00 2001 From: Paul Nechifor Date: Thu, 27 Nov 2025 04:11:09 +0200 Subject: [PATCH 3/4] add ollama --- pyproject.toml | 1 + 1 file changed, 1 insertion(+) diff --git a/pyproject.toml b/pyproject.toml index 5ce84983b9..341653e2ec 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -61,6 +61,7 @@ dependencies = [ "langchain-huggingface>=0.3.1", "langchain-ollama>=0.3.10", "bitsandbytes>=0.48.2,<1.0", + "ollama>=0.6.0", # Class Extraction "pydantic", From 7ca46e7f119eff17db1cc64f7f065c27ff3f98a9 Mon Sep 17 00:00:00 2001 From: Paul Nechifor Date: Thu, 27 Nov 2025 04:17:22 +0200 Subject: [PATCH 4/4] linting --- dimos/agents2/agent.py | 2 +- dimos/robot/unitree_webrtc/unitree_go2_blueprints.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/dimos/agents2/agent.py b/dimos/agents2/agent.py index bbe2c14c65..7c37febec0 100644 --- a/dimos/agents2/agent.py +++ b/dimos/agents2/agent.py @@ -296,7 +296,7 @@ def _get_state() -> str: # history() builds our message history dynamically # ensures we include latest system state, but not old ones. - messages = self.history() + messages = self.history() # type: ignore[no-untyped-call] # Some LLMs don't work without any human messages. Add an initial one. if len(messages) == 1 and isinstance(messages[0], SystemMessage): diff --git a/dimos/robot/unitree_webrtc/unitree_go2_blueprints.py b/dimos/robot/unitree_webrtc/unitree_go2_blueprints.py index 591678522c..e1383ae0f5 100644 --- a/dimos/robot/unitree_webrtc/unitree_go2_blueprints.py +++ b/dimos/robot/unitree_webrtc/unitree_go2_blueprints.py @@ -125,7 +125,7 @@ standard, llm_agent( model="qwen3:8b", - provider=Provider.OLLAMA, + provider=Provider.OLLAMA, # type: ignore[attr-defined] ), _common_agentic, ).requirements( @@ -136,7 +136,7 @@ standard, llm_agent( model="Qwen/Qwen2.5-1.5B-Instruct", - provider=Provider.HUGGINGFACE, + provider=Provider.HUGGINGFACE, # type: ignore[attr-defined] ), _common_agentic, )