Agent refactor

.github/workflows/docker.yml

@@ -132,7 +132,9 @@ jobs:
     uses: ./.github/workflows/_docker-build-template.yml
     with:
       should-run: ${{
-        needs.check-changes.result == 'success' && ((needs.ros-python.result == 'success') || (needs.ros-python.result == 'skipped')) && (needs.check-changes.outputs.dev == 'true')
+        needs.check-changes.result == 'success' && 
+        (needs.check-changes.outputs.dev == 'true' ||
+         (needs.ros-python.result == 'success' && (needs.check-changes.outputs.python == 'true' || needs.check-changes.outputs.ros == 'true')))
         }}
       from-image: ghcr.io/dimensionalos/ros-python:${{ needs.ros-python.result == 'success' && needs.check-changes.outputs.branch-tag || 'dev' }}
       to-image: ghcr.io/dimensionalos/ros-dev:${{ needs.check-changes.outputs.branch-tag }}
@@ -142,6 +144,7 @@ jobs:
     needs: [check-changes, ros-dev]
     if: always()
     uses: ./.github/workflows/tests.yml
+    secrets: inherit
     with:
       should-run: ${{
         needs.check-changes.result == 'success' &&
@@ -150,12 +153,13 @@ jobs:
         needs.check-changes.outputs.tests == 'true'))
         }}
       cmd: "pytest && pytest -m ros" # run tests that depend on ros as well
-      dev-image: ros-dev:${{ needs.check-changes.outputs.dev == 'true' && needs.ros-dev.result == 'success' && needs.check-changes.outputs.branch-tag || 'dev' }}
+      dev-image: ros-dev:${{ (needs.check-changes.outputs.python == 'true' || needs.check-changes.outputs.dev == 'true' || needs.check-changes.outputs.ros == 'true') && needs.ros-dev.result == 'success' && needs.check-changes.outputs.branch-tag || 'dev' }}
 
   run-tests:
     needs: [check-changes, dev]
     if: always()
     uses: ./.github/workflows/tests.yml
+    secrets: inherit
     with:
       should-run: ${{
         needs.check-changes.result == 'success' &&
@@ -164,13 +168,14 @@ jobs:
         needs.check-changes.outputs.tests == 'true'))
         }}
       cmd: "pytest"
-      dev-image: dev:${{ needs.check-changes.outputs.dev == 'true' && needs.dev.result == 'success' && needs.check-changes.outputs.branch-tag || 'dev' }}
+      dev-image: dev:${{ (needs.check-changes.outputs.python == 'true' || needs.check-changes.outputs.dev == 'true') && needs.dev.result == 'success' && needs.check-changes.outputs.branch-tag || 'dev' }}
 
   # we run in parallel with normal tests for speed
   run-heavy-tests:
     needs: [check-changes, dev]
     if: always()
     uses: ./.github/workflows/tests.yml
+    secrets: inherit
     with:
       should-run: ${{
         needs.check-changes.result == 'success' &&
@@ -179,12 +184,13 @@ jobs:
         needs.check-changes.outputs.tests == 'true'))
         }}
       cmd: "pytest -m heavy"
-      dev-image: dev:${{ needs.check-changes.outputs.dev == 'true' && needs.dev.result == 'success' && needs.check-changes.outputs.branch-tag || 'dev' }}
+      dev-image: dev:${{ (needs.check-changes.outputs.python == 'true' || needs.check-changes.outputs.dev == 'true') && needs.dev.result == 'success' && needs.check-changes.outputs.branch-tag || 'dev' }}
 
   run-lcm-tests:
     needs: [check-changes, dev]
     if: always()
     uses: ./.github/workflows/tests.yml
+    secrets: inherit
     with:
       should-run: ${{
         needs.check-changes.result == 'success' &&
@@ -193,7 +199,7 @@ jobs:
         needs.check-changes.outputs.tests == 'true'))
         }}
       cmd: "pytest -m lcm"
-      dev-image: dev:${{ needs.check-changes.outputs.dev == 'true' && needs.dev.result == 'success' && needs.check-changes.outputs.branch-tag || 'dev' }}
+      dev-image: dev:${{ (needs.check-changes.outputs.python == 'true' || needs.check-changes.outputs.dev == 'true') && needs.dev.result == 'success' && needs.check-changes.outputs.branch-tag || 'dev' }}
 
   # Run module tests directly to avoid pytest forking issues
   # run-module-tests:

.github/workflows/tests.yml

@@ -40,6 +40,10 @@ jobs:
     runs-on: [self-hosted, Linux]
     container:
       image: ghcr.io/dimensionalos/${{ inputs.dev-image }}
+      env:
+        OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
+        ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }}
+        ALIBABA_API_KEY: ${{ secrets.ALIBABA_API_KEY }}
 
     steps:
       - uses: actions/checkout@v4

dimos/agents/agent_message.py

@@ -0,0 +1,101 @@
+# Copyright 2025 Dimensional Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""AgentMessage type for multimodal agent communication."""
+
+from dataclasses import dataclass, field
+from typing import List, Optional, Union
+import time
+
+from dimos.msgs.sensor_msgs.Image import Image
+from dimos.agents.agent_types import AgentImage
+
+
+@dataclass
+class AgentMessage:
+    """Message type for agent communication with text and images.
+
+    This type supports multimodal messages containing both text strings
+    and AgentImage objects (base64 encoded) for vision-enabled agents.
+
+    The messages field contains multiple text strings that will be combined
+    into a single message when sent to the LLM.
+    """
+
+    messages: List[str] = field(default_factory=list)
+    images: List[AgentImage] = field(default_factory=list)
+    sender_id: Optional[str] = None
+    timestamp: float = field(default_factory=time.time)
+
+    def add_text(self, text: str) -> None:
+        """Add a text message."""
+        if text:  # Only add non-empty text
+            self.messages.append(text)
+
+    def add_image(self, image: Union[Image, AgentImage]) -> None:
+        """Add an image. Converts Image to AgentImage if needed."""
+        if isinstance(image, Image):
+            # Convert to AgentImage
+            agent_image = AgentImage(
+                base64_jpeg=image.agent_encode(),
+                width=image.width,
+                height=image.height,
+                metadata={"format": image.format.value, "frame_id": image.frame_id},
+            )
+            self.images.append(agent_image)
+        elif isinstance(image, AgentImage):
+            self.images.append(image)
+        else:
+            raise TypeError(f"Expected Image or AgentImage, got {type(image)}")
+
+    def has_text(self) -> bool:
+        """Check if message contains text."""
+        # Check if we have any non-empty messages
+        return any(msg for msg in self.messages if msg)
+
+    def has_images(self) -> bool:
+        """Check if message contains images."""
+        return len(self.images) > 0
+
+    def is_multimodal(self) -> bool:
+        """Check if message contains both text and images."""
+        return self.has_text() and self.has_images()
+
+    def get_primary_text(self) -> Optional[str]:
+        """Get the first text message, if any."""
+        return self.messages[0] if self.messages else None
+
+    def get_primary_image(self) -> Optional[AgentImage]:
+        """Get the first image, if any."""
+        return self.images[0] if self.images else None
+
+    def get_combined_text(self) -> str:
+        """Get all text messages combined into a single string."""
+        # Filter out any empty strings and join
+        return " ".join(msg for msg in self.messages if msg)
+
+    def clear(self) -> None:
+        """Clear all content."""
+        self.messages.clear()
+        self.images.clear()
+
+    def __repr__(self) -> str:
+        """String representation."""
+        return (
+            f"AgentMessage("
+            f"texts={len(self.messages)}, "
+            f"images={len(self.images)}, "
+            f"sender='{self.sender_id}', "
+            f"timestamp={self.timestamp})"
+        )