From 85a3b80874ce62f0a929622849475b5c9a732427 Mon Sep 17 00:00:00 2001
From: Kalyan Chakravarthy <chakravarthik27@gmail.com>
Date: Fri, 12 Apr 2024 12:30:32 +0530
Subject: [PATCH 1/2] Enhancements for user prompt handling for multi-dataset
 testing in Harness(langtest.py)

---
 langtest/langtest.py | 16 ++++++++++++++++
 1 file changed, 16 insertions(+)

diff --git a/langtest/langtest.py b/langtest/langtest.py
index b3f5b614e..2ca0a6778 100644
--- a/langtest/langtest.py
+++ b/langtest/langtest.py
@@ -1572,8 +1572,18 @@ def __multi_datasets_run(
     ):
         generated_results = {}
 
+        # temp_store_prompt
+        temp_store_prompt = self._config.get("model_parameters", {}).get(
+            "user_prompt", None
+        )
+
         # Run the testcases for each dataset
         for dataset_name, samples in testcases.items():
+            # update user prompt for each dataset
+            if temp_store_prompt:
+                self._config.get("model_parameters", {}).update(
+                    {"user_prompt": temp_store_prompt.get(dataset_name)}
+                )
             # Get the raw data for the dataset
             if isinstance(self.data, dict):
                 raw_data = self.data.get(dataset_name)
@@ -1597,6 +1607,12 @@ def __multi_datasets_run(
 
             print(f"{'':-^80}\n")
 
+        # resore user prompt
+        if temp_store_prompt:
+            self._config.get("model_parameters", {}).update(
+                {"user_prompt": temp_store_prompt}
+            )
+
         if (
             self.is_multi_dataset
             and self._generated_results is None

From c9055cfcf40fddf1d5e9ac0389ef0c05b876ac39 Mon Sep 17 00:00:00 2001
From: Kalyan Chakravarthy <chakravarthik27@gmail.com>
Date: Fri, 12 Apr 2024 14:00:32 +0530
Subject: [PATCH 2/2] Update user prompt handling in Harness for multi-dataset
 testing

---
 langtest/langtest.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/langtest/langtest.py b/langtest/langtest.py
index 2ca0a6778..b5ba0d90f 100644
--- a/langtest/langtest.py
+++ b/langtest/langtest.py
@@ -1580,7 +1580,7 @@ def __multi_datasets_run(
         # Run the testcases for each dataset
         for dataset_name, samples in testcases.items():
             # update user prompt for each dataset
-            if temp_store_prompt:
+            if temp_store_prompt and isinstance(temp_store_prompt, dict):
                 self._config.get("model_parameters", {}).update(
                     {"user_prompt": temp_store_prompt.get(dataset_name)}
                 )