(feat)Rich logging

benchmarks/commit0/run_infer.py

@@ -14,6 +14,7 @@
 )
 from benchmarks.commit0.config import INFER_DEFAULTS
 from benchmarks.utils.args_parser import get_parser
+from benchmarks.utils.console_logging import summarize_instance
 from benchmarks.utils.constants import EVAL_AGENT_SERVER_IMAGE
 from benchmarks.utils.conversation import build_event_persistence_callback
 from benchmarks.utils.critics import create_critic
@@ -507,6 +508,14 @@ def evaluate_instance(
         # Final debug log
         logger.info(f"Final eval_result: {eval_result}")
 
+        # Log instance summary
+        summarize_instance(
+            instance_id=instance.id,
+            conversation=conversation,
+            git_patch=git_patch or "",
+            logger=logger,
+        )
+
         # Save workspace as zip (if supported by workspace implementation)
         zip_dest = os.path.join(
             self.metadata.eval_output_dir, "repos", repo_name, f"{repo_name}.zip"

benchmarks/gaia/run_infer.py

@@ -14,6 +14,7 @@
 
 from benchmarks.gaia.config import INFER_DEFAULTS
 from benchmarks.gaia.scorer import question_scorer
+from benchmarks.utils.console_logging import summarize_instance
 from benchmarks.gaia.utils import image_to_jpg_base64_url, image_to_png_base64_url
 from benchmarks.utils.args_parser import get_parser
 from benchmarks.utils.constants import EVAL_AGENT_SERVER_IMAGE
@@ -364,6 +365,12 @@ def evaluate_instance(
             f"model_answer='{model_answer}', ground_truth='{ground_truth}'"
         )
 
+        summarize_instance(
+            instance_id=instance.id,
+            conversation=conversation,
+            logger=logger,
+        )
+
         # Collect history
 
         # Return evaluation output

benchmarks/multiswebench/run_infer.py

@@ -15,6 +15,7 @@
 from benchmarks.multiswebench.scripts.data.data_change import format_data_for_inference
 from benchmarks.utils.args_parser import get_parser
 from benchmarks.utils.build_utils import build_image
+from benchmarks.utils.console_logging import summarize_instance
 from benchmarks.utils.constants import EVAL_AGENT_SERVER_IMAGE
 from benchmarks.utils.conversation import build_event_persistence_callback
 from benchmarks.utils.critics import create_critic
@@ -399,6 +400,14 @@ def evaluate_instance(
         )
         git_patch = git_patch_result.stdout
 
+        # Log instance summary
+        summarize_instance(
+            instance_id=instance.id,
+            conversation=conversation,
+            git_patch=git_patch or "",
+            logger=logger,
+        )
+
         # EvalOutput is your model; keep fields consistent with prior JSONL
         out = EvalOutput(
             instance_id=instance.id,

benchmarks/openagentsafety/run_infer.py

@@ -14,6 +14,7 @@
 
 from benchmarks.openagentsafety.build_images import build_workspace_image
 from benchmarks.utils.args_parser import get_parser
+from benchmarks.utils.console_logging import summarize_instance
 from benchmarks.utils.conversation import build_event_persistence_callback
 from benchmarks.utils.critics import create_critic
 from benchmarks.utils.dataset import get_dataset
@@ -513,6 +514,12 @@ def event_callback(event) -> None:
             logger.warning(f"No evaluator_code for {instance.id}")
             eval_result = {"error": "No evaluator code provided"}
 
+        summarize_instance(
+            instance_id=instance.id,
+            conversation=conversation,
+            logger=logger,
+        )
+
         # Collect cost metrics from LLM
         metrics = None
         if hasattr(self.metadata.llm, "metrics"):

benchmarks/swebench/run_infer.py

@@ -16,6 +16,7 @@
 from benchmarks.utils.args_parser import get_parser
 from benchmarks.utils.build_utils import build_image
 from benchmarks.utils.constants import EVAL_AGENT_SERVER_IMAGE
+from benchmarks.utils.console_logging import summarize_instance
 from benchmarks.utils.conversation import build_event_persistence_callback
 from benchmarks.utils.critics import create_critic
 from benchmarks.utils.dataset import get_dataset
@@ -306,6 +307,14 @@ def evaluate_instance(
         )
         git_patch = git_patch_result.stdout
 
+        # Log instance summary
+        summarize_instance(
+            instance_id=instance.id,
+            conversation=conversation,
+            git_patch=git_patch,
+            logger=logger,
+        )
+
         # EvalOutput is your model; keep fields consistent with prior JSONL
         out = EvalOutput(
             instance_id=instance.id,

benchmarks/swebenchmultimodal/run_infer.py

@@ -13,6 +13,7 @@
 from benchmarks.swebenchmultimodal.config import INFER_DEFAULTS
 from benchmarks.utils.args_parser import get_parser
 from benchmarks.utils.build_utils import build_image
+from benchmarks.utils.console_logging import summarize_instance
 from benchmarks.utils.constants import EVAL_AGENT_SERVER_IMAGE
 from benchmarks.utils.conversation import build_event_persistence_callback
 from benchmarks.utils.critics import create_critic
@@ -394,6 +395,14 @@ def evaluate_instance(
         )
         git_patch = git_patch_result.stdout
 
+        # Log instance summary
+        summarize_instance(
+            instance_id=instance.id,
+            conversation=conversation,
+            git_patch=git_patch or "",
+            logger=logger,
+        )
+
         # EvalOutput is your model; keep fields consistent with prior JSONL
         out = EvalOutput(
             instance_id=instance.id,

benchmarks/swtbench/run_infer.py

@@ -7,6 +7,7 @@
 
 from benchmarks.swtbench.config import INFER_DEFAULTS
 from benchmarks.utils.args_parser import get_parser
+from benchmarks.utils.console_logging import summarize_instance
 from benchmarks.utils.constants import EVAL_AGENT_SERVER_IMAGE
 from benchmarks.utils.conversation import build_event_persistence_callback
 from benchmarks.utils.critics import create_critic
@@ -327,6 +328,14 @@ def evaluate_instance(
         )
         git_patch = git_patch_result.stdout
 
+        # Log instance summary
+        summarize_instance(
+            instance_id=instance.id,
+            conversation=conversation,
+            git_patch=git_patch or "",
+            logger=logger,
+        )
+
         out = EvalOutput(
             instance_id=instance.id,
             attempt=self.current_attempt,