adding timestamp knowledge to agent prompt and info

dimos/perception/experimental/temporal_memory/temporal_utils/graph_utils.py

@@ -129,6 +129,7 @@ def build_graph_context(
             "relationships": [],
             "spatial_info": [],
             "semantic_knowledge": [],
+            "entity_timestamps": [],
         }
 
         # Convert time_window_s to a (start_ts, end_ts) tuple if provided
@@ -143,6 +144,25 @@ def build_graph_context(
                 ref_time = max((e.get("last_seen_ts", 0) for e in all_entities), default=0)
             time_window_tuple = (max(0, ref_time - time_window_s), ref_time)
 
+        # Get entity timestamp information for visibility duration queries
+        for entity_id in entity_ids:
+            entity = graph_db.get_entity(entity_id)
+            if entity:
+                first_seen = entity.get("first_seen_ts")
+                last_seen = entity.get("last_seen_ts")
+                duration_s = None
+                if first_seen is not None and last_seen is not None:
+                    duration_s = last_seen - first_seen
+
+                graph_context["entity_timestamps"].append(
+                    {
+                        "entity_id": entity_id,
+                        "first_seen_ts": first_seen,
+                        "last_seen_ts": last_seen,
+                        "duration_s": duration_s,
+                    }
+                )
+
         # Get recent relationships for each entity
         for entity_id in entity_ids:
             # Get relationships (Graph 1: interactions)

dimos/perception/experimental/temporal_memory/temporal_utils/prompts.py

@@ -240,9 +240,15 @@ def build_query_prompt(
 - {currently_present_str}
 - The 'entity_roster' contains all known entities with their descriptions
 - The 'rolling_summary' describes what has happened over time
+- The 'graph_knowledge.entity_timestamps' contains visibility information for each entity:
+  - 'first_seen_ts': timestamp (in seconds) when the entity was first detected
+  - 'last_seen_ts': timestamp (in seconds) when the entity was last detected
+  - 'duration_s': total time span from first to last appearance (last_seen_ts - first_seen_ts)
+  - Use this information to answer questions about when entities appeared, disappeared, or how long they were visible
 - If 'currently_present_entities' is empty, it means no entities were detected in recent windows, but entities may still exist in the roster from earlier
-- Answer based on the provided context (entity_roster, rolling_summary, currently_present_entities) AND what you see in the current frame
+- Answer based on the provided context (entity_roster, rolling_summary, currently_present_entities, graph_knowledge) AND what you see in the current frame
 - If the context says entities were present but you don't see them in the current frame, mention both: what was recently detected AND what you currently see
+- For duration questions, use the 'duration_s' field from 'entity_timestamps' if available
 
 Provide a concise answer.
 """