Add OpenAI benchmark sweep and analysis tooling

codeclash/analysis/metrics/elo.py

@@ -14,7 +14,7 @@
 from tqdm import tqdm
 
 from codeclash.analysis.significance import calculate_p_value
-from codeclash.analysis.viz.utils import ASSETS_DIR, FONT_BOLD, MODEL_TO_DISPLAY_NAME
+from codeclash.analysis.viz.utils import ASSETS_DIR, FONT_BOLD, MODEL_TO_DISPLAY_NAME, model_display_name
 from codeclash.constants import LOCAL_LOG_DIR, RESULT_TIE
 from codeclash.utils.log import add_file_handler, get_logger
 
@@ -75,9 +75,6 @@ def __init__(
             lambda: defaultdict(list)
         )
 
-    def _get_unique_model_name(self, model: str) -> str:
-        return model.rpartition("/")[2]
-
     def _get_sorted_pair(self, p1: str, p2: str) -> tuple[str, str]:
         return tuple(sorted([p1, p2]))
 
@@ -154,8 +151,6 @@ def _process_tournament(self, metadata_path: Path) -> None:
             return
 
         player_names = [p["name"] for p in players]
-        models = [p["config"]["model"]["model_name"].strip("@") for p in players]
-
         # Aggregate scores for each round
         p1_round_scores = []
         p2_round_scores = []
@@ -199,7 +194,7 @@ def _process_tournament(self, metadata_path: Path) -> None:
             p2_score = sum(p2_round_scores)
 
         # Convert to unique names and sorted pair when updating matrix
-        unique_names = [self._get_unique_model_name(m) for m in models]
+        unique_names = player_names
         sorted_pair = self._get_sorted_pair(unique_names[0], unique_names[1])
 
         if unique_names[0] == sorted_pair[0]:
@@ -550,7 +545,7 @@ def create_elo_plots(self, output_dir: Path) -> None:
         player_order = [all_players[i] for i in all_indices]
 
         # Translate to display names
-        display_names = [MODEL_TO_DISPLAY_NAME.get(p, p) for p in player_order]
+        display_names = [model_display_name(p) for p in player_order]
 
         # Create mapping from player to y-position
         player_to_pos = {p: i for i, p in enumerate(player_order)}
@@ -698,7 +693,7 @@ def create_validation_plots(self, output_dir: Path, regularization: float = 0.01
 
                 ax.set_xlabel("BT Strength", fontproperties=FONT_BOLD, fontsize=12)
                 ax.set_ylabel("Negative Log-Likelihood", fontproperties=FONT_BOLD, fontsize=12)
-                display_name = MODEL_TO_DISPLAY_NAME.get(player, player)
+                display_name = model_display_name(player)
                 ax.set_title(display_name, fontproperties=FONT_BOLD, fontsize=14)
                 legend = ax.legend(prop=FONT_BOLD, fontsize=10, loc="upper right")
                 legend.set_frame_on(False)
@@ -777,7 +772,7 @@ def _create_rank_matrix_plot(
         rank_matrix = (rank_matrix / self.n_bootstrap) * 100
 
         # Translate player names to display names
-        display_names = [MODEL_TO_DISPLAY_NAME.get(p, p) for p in players]
+        display_names = [model_display_name(p) for p in players]
 
         fig, ax = plt.subplots(figsize=(6, 6))
         im = ax.imshow(rank_matrix, cmap="YlOrRd", aspect="auto", vmin=0, vmax=100)
@@ -826,7 +821,7 @@ def _create_elo_violin_plot(
         elo_data = [elo_samples[p] for p in players]
 
         # Translate player names to display names
-        display_names = [MODEL_TO_DISPLAY_NAME.get(p, p) for p in players]
+        display_names = [model_display_name(p) for p in players]
 
         fig, ax = plt.subplots(figsize=(6, 6))
 
@@ -1095,7 +1090,7 @@ def _plot_results(self, results_by_max_round: dict[int, dict[str, dict[str, floa
                             elos_list.append(results_by_max_round[max_round][game_name][player])
 
                 if max_rounds_list:
-                    display_name = MODEL_TO_DISPLAY_NAME.get(player, player)
+                    display_name = model_display_name(player)
                     ax.plot(max_rounds_list, elos_list, marker="o", label=display_name, linewidth=2, markersize=6)
 
             ax.set_xlabel("Max Round", fontproperties=FONT_BOLD, fontsize=14)
@@ -1212,7 +1207,7 @@ def _plot_results(self, results_by_round: dict[int, dict[str, dict[str, float]]]
                             elos_list.append(results_by_round[round_num][game_name][player])
 
                 if rounds_list:
-                    display_name = MODEL_TO_DISPLAY_NAME.get(player, player)
+                    display_name = model_display_name(player)
                     ax.plot(rounds_list, elos_list, marker="o", label=display_name, linewidth=2, markersize=6)
 
             ax.set_xlabel("Round", fontproperties=FONT_BOLD, fontsize=14)
@@ -1348,7 +1343,7 @@ def write_latex_table(results: dict[str, dict], output_dir: Path) -> None:
     lines.append(r"\midrule")
 
     for player, all_elo in sorted_players:
-        display_name = MODEL_TO_DISPLAY_NAME.get(player, player)
+        display_name = model_display_name(player)
         row_parts = [display_name.replace("_", r"\_")]
 
         for game_name in games_in_table:
@@ -1407,7 +1402,7 @@ def write_website_results(results: dict[str, dict], output_dir: Path) -> None:
         # Create leaderboard entries
         board = []
         for rank, (player, elo) in enumerate(sorted_players):
-            entry = {"rank": rank + 1, "model": MODEL_TO_DISPLAY_NAME.get(player, player), "elo": int(round(elo))}
+            entry = {"rank": rank + 1, "model": model_display_name(player), "elo": int(round(elo))}
             # Add confidence interval if available
             if elo_std is not None:
                 player_idx = players.index(player)
@@ -1506,7 +1501,7 @@ def write_latex_table_plain(results: dict[str, dict], output_dir: Path) -> None:
     lines.append(r"\midrule")
 
     for player, all_elo in sorted_players:
-        display_name = MODEL_TO_DISPLAY_NAME.get(player, player)
+        display_name = model_display_name(player)
         row_parts = [display_name.replace("_", r"\_")]
 
         for game_name in games_in_table:

codeclash/analysis/metrics/win_rate.py

@@ -31,16 +31,16 @@ def main(log_dir: Path):
     model_profiles = {}
     for game_log_folder in tqdm([x.parent for x in log_dir.rglob("metadata.json")]):
         game_id = game_log_folder.name.split(".")[1]
-        player_ids = [x.name for x in (game_log_folder / "players").iterdir() if x.is_dir()]
         metadata = json.load(open(game_log_folder / "metadata.json"))
         try:
-            player_to_model = {
-                x["name"]: x["config"]["model"]["model_name"].strip("@").split("/")[-1]
-                for x in metadata["config"]["players"]
-            }
+            player_ids = [x["name"] for x in metadata["config"]["players"]]
+            player_to_model = {x["name"]: x["name"] for x in metadata["config"]["players"]}
         except KeyError:
             continue
-        num_rounds = len(metadata["round_stats"])
+        round_stats = metadata.get("round_stats")
+        if not isinstance(round_stats, dict) or not round_stats:
+            continue
+        num_rounds = len(round_stats)
 
         # Only count each unique model once per game
         unique_models = {player_to_model[player] for player in player_ids}
@@ -55,7 +55,7 @@ def main(log_dir: Path):
                     player_id=player_id, model_name=model_name, game_id=game_id, count=num_rounds
                 )
 
-        for round, details in metadata["round_stats"].items():
+        for round, details in round_stats.items():
             if round == "0":
                 # Skip initial round
                 continue

codeclash/analysis/viz/heatmap_win_rates.py

@@ -59,7 +59,7 @@ def main(log_dir: Path, unit: str = "rounds", output_file: Path = ASSETS_DIR / "
 
     # Build matrix
     models = sorted({m for pair in results.keys() for m in pair})
-    clean_names = [MODEL_TO_DISPLAY_NAME[m.split("/")[-1]] for m in models]
+    clean_names = [MODEL_TO_DISPLAY_NAME.get(m.split("/")[-1], m.split("/")[-1]) for m in models]
     n = len(models)
 
     matrix = np.full((n, n), np.nan)
@@ -73,7 +73,8 @@ def main(log_dir: Path, unit: str = "rounds", output_file: Path = ASSETS_DIR / "
         total_wins = sum(results[(m1, m2)][0] for m2 in models if m1 != m2)
         total_matches = sum(results[(m1, m2)][1] for m2 in models if m1 != m2)
         avg_win_rate = total_wins / total_matches if total_matches > 0 else 0
-        print(f"{MODEL_TO_DISPLAY_NAME[m1.split('/')[-1]]}: {avg_win_rate:.2%} win rate over {total_matches} matches")
+        label = MODEL_TO_DISPLAY_NAME.get(m1.split("/")[-1], m1.split("/")[-1])
+        print(f"{label}: {avg_win_rate:.2%} win rate over {total_matches} matches")
 
     # Plot
     FONT_BOLD.set_size(18)

codeclash/analysis/viz/utils.py

@@ -21,6 +21,22 @@
     "o3": "o3",
 }
 
+
+def model_display_name(model: str) -> str:
+    label = MODEL_TO_DISPLAY_NAME.get(model, model)
+    tier_labels = {
+        "-default": " (Default)",
+        "-low": " (Low)",
+        "-medium": " (Medium)",
+        "-high": " (High)",
+    }
+    for suffix, pretty in tier_labels.items():
+        if model.endswith(suffix):
+            base = model[: -len(suffix)]
+            base_label = MODEL_TO_DISPLAY_NAME.get(base, base)
+            return f"{base_label}{pretty}"
+    return label
+
 MODEL_TO_COLOR = {
     "anthropic/claude-sonnet-4-20250514": "#FFD449",
     "anthropic/claude-sonnet-4-5-20250929": "#F75C03",

codeclash/arenas/huskybench/HuskyBench.Dockerfile

@@ -16,5 +16,6 @@ RUN git clone https://github.com/CodeClash-ai/HuskyBench.git /workspace \
     && git remote set-url origin https://github.com/CodeClash-ai/HuskyBench.git
 WORKDIR /workspace
 
-RUN pip install -r engine/requirements.txt
+RUN pip install --no-cache-dir Cython setuptools wheel \
+    && pip install --no-cache-dir -r engine/requirements.txt
 RUN mkdir -p /workspace/engine/output

codeclash/arenas/robocode/robocode.py

@@ -10,6 +10,7 @@
 
 from codeclash.agents.player import Player
 from codeclash.arenas.arena import CodeArena, RoundStats
+from codeclash.constants import RESULT_TIE
 from codeclash.utils.environment import create_file_in_container
 
 RC_FILE = Path("MyTank.java")
@@ -140,7 +141,13 @@ def get_results(self, agents: list[Player], round_num: int, stats: RoundStats):
                     player = match.group(2).rsplit(".", 1)[0]
                     scores[player] += int(match.group(3))
 
-        stats.winner = max(scores, key=scores.get)
+        if not scores:
+            stats.winner = RESULT_TIE
+            return
+
+        max_score = max(scores.values())
+        leaders = [player for player, score in scores.items() if score == max_score]
+        stats.winner = RESULT_TIE if len(leaders) > 1 else leaders[0]
         stats.scores = scores
         for player, score in scores.items():
             stats.player_stats[player].score = score