From e912b0298cd3fb14325b5545bcf230422ea05524 Mon Sep 17 00:00:00 2001 From: arcticfly Date: Fri, 19 Dec 2025 13:02:15 -0800 Subject: [PATCH 1/3] Log metrics horizontally in W&B to simplify comparison to future runs --- .../log_constant_metrics_wandb.py | 44 +++++++++++++++++++ 1 file changed, 44 insertions(+) create mode 100644 src/art/utils/benchmarking/log_constant_metrics_wandb.py diff --git a/src/art/utils/benchmarking/log_constant_metrics_wandb.py b/src/art/utils/benchmarking/log_constant_metrics_wandb.py new file mode 100644 index 000000000..2337025a5 --- /dev/null +++ b/src/art/utils/benchmarking/log_constant_metrics_wandb.py @@ -0,0 +1,44 @@ +"""Utilities for logging constant baseline metrics to Weights & Biases.""" + +import art +import wandb + + +async def log_constant_metrics_wandb( + model: art.Model, + num_steps: int, + split: str, + metrics: dict[str, float], +) -> None: + """ + Log constant metrics to W&B as horizontal lines across all training steps. + + Creates a W&B run and logs the same values at every step from 0 to + `num_steps`, producing horizontal reference lines on charts. Useful for + comparing training curves against static baselines. + + Parameters + ---------- + model : art.Model + The model whose `project` and `name` are used for the W&B run. + num_steps : int + Total training steps. Metrics are logged at steps 0 through `num_steps`. + split : str + Data split name (e.g., "val"). Used as prefix: "{split}/{metric_name}". + metrics : dict[str, float] + Metric names mapped to their constant values. + """ + run = wandb.init( + project=model.project, + name=model.name, + reinit="create_new", + ) + + # Prefix metrics with split + prefixed_metrics = {f"{split}/{key}": value for key, value in metrics.items()} + + # Log at every step to create a horizontal line + for step in range(num_steps + 1): + run.log(prefixed_metrics, step=step) + + run.finish() From 48618be224fcbaa8c5329051d476eac8a34d5a82 Mon Sep 17 00:00:00 2001 From: arcticfly Date: Fri, 19 Dec 2025 13:06:41 -0800 Subject: [PATCH 2/3] Fix lint checks --- src/art/utils/benchmarking/log_constant_metrics_wandb.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/art/utils/benchmarking/log_constant_metrics_wandb.py b/src/art/utils/benchmarking/log_constant_metrics_wandb.py index 2337025a5..e96044ab6 100644 --- a/src/art/utils/benchmarking/log_constant_metrics_wandb.py +++ b/src/art/utils/benchmarking/log_constant_metrics_wandb.py @@ -1,8 +1,9 @@ """Utilities for logging constant baseline metrics to Weights & Biases.""" -import art import wandb +import art + async def log_constant_metrics_wandb( model: art.Model, From d48b0f61223fc342362f2371f6cb54166884a275 Mon Sep 17 00:00:00 2001 From: arcticfly Date: Fri, 19 Dec 2025 13:16:49 -0800 Subject: [PATCH 3/3] Accept split_metrics --- .../log_constant_metrics_wandb.py | 22 +++++++++++-------- 1 file changed, 13 insertions(+), 9 deletions(-) diff --git a/src/art/utils/benchmarking/log_constant_metrics_wandb.py b/src/art/utils/benchmarking/log_constant_metrics_wandb.py index e96044ab6..7f876a191 100644 --- a/src/art/utils/benchmarking/log_constant_metrics_wandb.py +++ b/src/art/utils/benchmarking/log_constant_metrics_wandb.py @@ -8,8 +8,7 @@ async def log_constant_metrics_wandb( model: art.Model, num_steps: int, - split: str, - metrics: dict[str, float], + split_metrics: dict[str, dict[str, float]], ) -> None: """ Log constant metrics to W&B as horizontal lines across all training steps. @@ -24,10 +23,11 @@ async def log_constant_metrics_wandb( The model whose `project` and `name` are used for the W&B run. num_steps : int Total training steps. Metrics are logged at steps 0 through `num_steps`. - split : str - Data split name (e.g., "val"). Used as prefix: "{split}/{metric_name}". - metrics : dict[str, float] - Metric names mapped to their constant values. + split_metrics : dict[str, dict[str, float]] + Nested dict mapping split names (e.g., "train", "val") to metric dicts. + Each metric is logged as "{split}/{metric_name}". + + Example: `{"train": {"loss": 0.5}, "val": {"loss": 0.4, "accuracy": 0.8}}` """ run = wandb.init( project=model.project, @@ -35,10 +35,14 @@ async def log_constant_metrics_wandb( reinit="create_new", ) - # Prefix metrics with split - prefixed_metrics = {f"{split}/{key}": value for key, value in metrics.items()} + # Prefix metrics with their split names + prefixed_metrics = { + f"{split}/{key}": value + for split, metrics in split_metrics.items() + for key, value in metrics.items() + } - # Log at every step to create a horizontal line + # Log at every step to create horizontal lines for step in range(num_steps + 1): run.log(prefixed_metrics, step=step)