From 0d2b38fdc35d42f14c1d1db4e616752b2e9c34ac Mon Sep 17 00:00:00 2001
From: Qian-Cheng-nju <846853688@qq.com>
Date: Sun, 9 Nov 2025 22:34:36 +0800
Subject: [PATCH 1/2] feat: Distinguish evaluator and model API keys in
 env.toml

---
 sdk/utils.py | 30 ++++++++++++++++++++++++++++--
 1 file changed, 28 insertions(+), 2 deletions(-)

diff --git a/sdk/utils.py b/sdk/utils.py
index 616b56c9..bcbf45f3 100644
--- a/sdk/utils.py
+++ b/sdk/utils.py
@@ -33,10 +33,26 @@ def set_llm_endpoint_from_config(config_path):
         sys.exit(1)
     logger.info('Loaded configuration:')
 
-    # read all vaules in the llm section and set them as environment variables
+    # Read all values in the llm section and set them as environment variables
     llm_config = config.get('llm', {})
+    evaluator_config = config.get('evaluator_api_keys', {})
 
-    logger.info('Setting the following environment variables:')
+    # Detect conflicts between [llm] and [evaluator_api_keys]
+    common_keys = set(llm_config.keys()) & set(evaluator_config.keys())
+    if common_keys:
+        conflicting_keys = []
+        for key in common_keys:
+            if llm_config[key] != evaluator_config[key]:
+                conflicting_keys.append(key)
+
+        if conflicting_keys:
+            logger.warning('Warning: The following API keys are defined in both [llm] and [evaluator_api_keys] with different values:')
+            for key in conflicting_keys:
+                logger.warning('  - %s', key)
+            logger.warning('Only [evaluator_api_keys] values will be used for both evaluator and model under test.')
+
+    # First, set environment variables from [llm]
+    logger.info('Setting the following environment variables from [llm]:')
     for key, value in llm_config.items():
         logger.info('%s', f'{key}: [REDACTED]' if 'key' in key.lower() else f'{key}: {value}')
         os.environ[key] = value
@@ -44,3 +60,13 @@ def set_llm_endpoint_from_config(config_path):
         if key == 'AZURE_API_KEY':
             os.environ['AZURE_OPENAI_API_KEY'] = value
             logger.info('AZURE_OPENAI_API_KEY: [REDACTED]')
+
+    # Then, set environment variables from [evaluator_api_keys] (will override [llm] if conflict)
+    logger.info('Setting the following environment variables from [evaluator_api_keys]:')
+    for key, value in evaluator_config.items():
+        logger.info('%s', f'{key}: [REDACTED]' if 'key' in key.lower() else f'{key}: {value}')
+        os.environ[key] = value
+        # add exception for SWE-Agent:
+        if key == 'AZURE_API_KEY':
+            os.environ['AZURE_OPENAI_API_KEY'] = value
+            logger.info('AZURE_OPENAI_API_KEY: [REDACTED]')

From b2acaa7012257c85747768bc73ba571862e184a2 Mon Sep 17 00:00:00 2001
From: Qian-Cheng-nju <846853688@qq.com>
Date: Sun, 9 Nov 2025 22:48:24 +0800
Subject: [PATCH 2/2] feat: Add validation for required evaluator API keys

---
 sdk/utils.py | 11 +++++++++++
 1 file changed, 11 insertions(+)

diff --git a/sdk/utils.py b/sdk/utils.py
index bcbf45f3..cbd79357 100644
--- a/sdk/utils.py
+++ b/sdk/utils.py
@@ -37,6 +37,17 @@ def set_llm_endpoint_from_config(config_path):
     llm_config = config.get('llm', {})
     evaluator_config = config.get('evaluator_api_keys', {})
 
+    # Validate evaluator_api_keys: if the section exists, all keys must have non-empty values
+    if evaluator_config:
+        empty_keys = [key for key, value in evaluator_config.items() if not value or str(value).strip() == '']
+        if empty_keys:
+            logger.error('Error: The following required API keys in [evaluator_api_keys] are empty:')
+            for key in empty_keys:
+                logger.error('  - %s', key)
+            logger.error('[evaluator_api_keys] section indicates these are REQUIRED for the benchmark to run.')
+            logger.error('Please provide valid API keys for all entries in [evaluator_api_keys].')
+            sys.exit(1)
+
     # Detect conflicts between [llm] and [evaluator_api_keys]
     common_keys = set(llm_config.keys()) & set(evaluator_config.keys())
     if common_keys: