Distinguish the models used in the executor and evaluator

sdk/utils.py

@@ -33,14 +33,51 @@ def set_llm_endpoint_from_config(config_path):
         sys.exit(1)
     logger.info('Loaded configuration:')
 
-    # read all vaules in the llm section and set them as environment variables
+    # Read all values in the llm section and set them as environment variables
     llm_config = config.get('llm', {})
+    evaluator_config = config.get('evaluator_api_keys', {})
 
-    logger.info('Setting the following environment variables:')
+    # Validate evaluator_api_keys: if the section exists, all keys must have non-empty values
+    if evaluator_config:
+        empty_keys = [key for key, value in evaluator_config.items() if not value or str(value).strip() == '']
+        if empty_keys:
+            logger.error('Error: The following required API keys in [evaluator_api_keys] are empty:')
+            for key in empty_keys:
+                logger.error('  - %s', key)
+            logger.error('[evaluator_api_keys] section indicates these are REQUIRED for the benchmark to run.')
+            logger.error('Please provide valid API keys for all entries in [evaluator_api_keys].')
+            sys.exit(1)
+
+    # Detect conflicts between [llm] and [evaluator_api_keys]
+    common_keys = set(llm_config.keys()) & set(evaluator_config.keys())
+    if common_keys:
+        conflicting_keys = []
+        for key in common_keys:
+            if llm_config[key] != evaluator_config[key]:
+                conflicting_keys.append(key)
+
+        if conflicting_keys:
+            logger.warning('Warning: The following API keys are defined in both [llm] and [evaluator_api_keys] with different values:')
+            for key in conflicting_keys:
+                logger.warning('  - %s', key)
+            logger.warning('Only [evaluator_api_keys] values will be used for both evaluator and model under test.')
+
+    # First, set environment variables from [llm]
+    logger.info('Setting the following environment variables from [llm]:')
     for key, value in llm_config.items():
         logger.info('%s', f'{key}: [REDACTED]' if 'key' in key.lower() else f'{key}: {value}')
         os.environ[key] = value
         # add exception for SWE-Agent:
         if key == 'AZURE_API_KEY':
             os.environ['AZURE_OPENAI_API_KEY'] = value
             logger.info('AZURE_OPENAI_API_KEY: [REDACTED]')
+
+    # Then, set environment variables from [evaluator_api_keys] (will override [llm] if conflict)
+    logger.info('Setting the following environment variables from [evaluator_api_keys]:')
+    for key, value in evaluator_config.items():
+        logger.info('%s', f'{key}: [REDACTED]' if 'key' in key.lower() else f'{key}: {value}')
+        os.environ[key] = value
+        # add exception for SWE-Agent:
+        if key == 'AZURE_API_KEY':
+            os.environ['AZURE_OPENAI_API_KEY'] = value
+            logger.info('AZURE_OPENAI_API_KEY: [REDACTED]')