partial implementation from release/2.8.0

.github/workflows/build_and_test.yml

@@ -17,7 +17,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        python-version: [ "3.9","3.10", "3.11" ]
+        python-version: ["3.12", "3.13" ]
 
     steps:
       - name: Free up disk space at start
@@ -53,7 +53,7 @@ jobs:
         if: steps.cached-poetry-dependencies.outputs.cache-hit != 'true'
         run: |
           poetry cache clear pypi --all -n > /dev/null
-          poetry install --with dev --all-extras --no-cache --quiet --no-interaction
+          poetry install --with dev --all-extras --no-cache --no-interaction
           source ./.venv/bin/activate && pip uninstall -y pyspark && rm -rf ./.venv/lib/python${{ matrix.python-version }}/site-packages/pyspark*/
           pip install pyspark==3.5.6
 

.github/workflows/llm_tests_build.yml

@@ -10,7 +10,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        python-version: [ "3.8", "3.9", "3.10" ]
+        python-version: [ "3.12", "3.13" ]
 
     steps:
       - uses: actions/checkout@v3

.github/workflows/release.yaml

@@ -9,7 +9,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        python-version: [3.9]
+        python-version: [3.12]
         poetry-version: [2.1.3]
         os: [ubuntu-latest]
     runs-on: ${{ matrix.os }}

langtest/datahandler/datasource.py

@@ -808,7 +808,7 @@ def load_raw_data(self, standardize_columns: bool = False) -> List[Dict]:
                 parsed CSV file into list of dicts
         """
 
-        if type(self._file_path) == dict:
+        if isinstance(self._file_path, dict):
             df = pd.read_csv(self._file_path["data_source"])
 
             if self.task == "text-classification":

langtest/embeddings/openai.py

@@ -10,15 +10,15 @@
 class OpenaiEmbeddings:
     LIB_NAME = "openai"
 
-    def __init__(self, model="text-embedding-ada-002"):
+    def __init__(self, model="text-embedding-3-small"):
         self.model = model
         self.api_key = os.environ.get("OPENAI_API_KEY")
         self.openai = None
         self._check_openai_package()
         if not self.api_key:
             raise ValueError(Errors.E032())
 
-        self.openai.api_key = self.api_key
+        # self.openai.api_key = self.api_key
 
     def _check_openai_package(self):
         """Check if the 'openai' package is installed and import the required functions.
@@ -44,13 +44,17 @@ def get_embedding(
             list[float]: A list of floating-point values representing the text's embedding.
         """
         if isinstance(text, list):
-            response = self.openai.Embedding.create(input=text, model=self.model)
+            response = self.openai.Client(api_key=self.api_key).embedding.create(
+                input=text, model=self.model
+            )
             embedding = [
-                np.array(response["data"][i]["embedding"]).reshape(1, -1)
+                np.array(response.data[i].embedding).reshape(1, -1)
                 for i in range(len(text))
             ]
             return embedding
         else:
-            response = self.openai.Embedding.create(input=[text], model=self.model)
-            embedding = np.array(response["data"][0]["embedding"]).reshape(1, -1)
+            response = self.openai.Client(api_key=self.api_key).embedding.create(
+                input=[text], model=self.model
+            )
+            embedding = np.array(response.data[0].embedding).reshape(1, -1)
             return embedding

langtest/langtest.py

@@ -286,7 +286,7 @@ def configure(self, config: Union[HarnessConfig, dict, str]) -> HarnessConfig:
         Returns:
             dict: Loaded configuration.
         """
-        if type(config) == dict:
+        if isinstance(config, dict):
             self._config = config
         else:
             with open(config, "r", encoding="utf-8") as yml:

langtest/metrics/llm_eval.py

@@ -66,25 +66,22 @@ def build_prompt(
                 f"""\n\nScore the student answer based on the following criteria:\n{eval_criteria}"""
             )
 
-        prompt += dedent(
-            f"""
-        Example Format:
-        QUESTION: question here
-        STUDENT ANSWER: student's answer here
-        TRUE ANSWER: true answer here
-        GRADE: {grade_list} here
-
-        {
-            ("Grade the student answers based ONLY on their factual accuracy. Ignore differences"
-             " in punctuation and phrasing between the student answer and true answer. It is OK "
-             "if the student answer contains more or relevant information than the true answer, as"
-             " long as it does not contain any conflicting statements. Begin!")
-        }
-
-        QUESTION: {{query}}
-        STUDENT ANSWER: {{result}}
-        TRUE ANSWER: {{answer}}
-        GRADE:"""
+        prompt += (
+            "Example Format:\n"
+            "QUESTION: question here\n"
+            "STUDENT ANSWER: student's answer here\n"
+            "TRUE ANSWER: true answer here\n"
+            f"GRADE: {grade_list} here"
+            "\n\n"
+            "Grade the student answers based ONLY on their factual accuracy. Ignore differences"
+            " in punctuation and phrasing between the student answer and true answer. It is OK "
+            "if the student answer contains more or relevant information than the true answer, as"
+            " long as it does not contain any conflicting statements. Begin!"
+            "\n\n"
+            "QUESTION: {{query}}\n"
+            "STUDENT ANSWER: {{result}}\n"
+            "TRUE ANSWER: {{answer}}\n"
+            "GRADE:\n"
         )
         return prompt
 

langtest/modelhandler/__init__.py

@@ -33,15 +33,15 @@
 
 
 if "langchain" in INSTALLED_HUBS:
-    import langchain
+    import langchain_classic
 
     LANGCHAIN_HUBS = {
         (
             RENAME_HUBS.get(hub.lower(), hub.lower())
             if hub.lower() in RENAME_HUBS
             else hub.lower()
         ): hub
-        for hub in langchain.llms.__all__
+        for hub in langchain_classic.llms.__all__
     }
     LANGCHAIN_HUBS["openrouter"] = "openrouter"
 

langtest/modelhandler/llm_modelhandler.py

@@ -3,9 +3,9 @@
 
 import os
 from typing import Any, List, Type, Union, TypeVar
-import langchain.llms as lc
+import langchain_classic.llms as lc
 import langchain.chat_models as chat_models
-from langchain.chains.llm import LLMChain
+from langchain_classic.chains.llm import LLMChain
 from langchain_core.prompts import PromptTemplate
 from langchain_core.language_models.base import BaseLanguageModel
 from langchain_core.exceptions import OutputParserException

langtest/modelhandler/modelhandler.py

@@ -14,16 +14,16 @@
 }
 
 if try_import_lib("langchain"):
-    import langchain
-    import langchain.llms
+    import langchain_classic
+    import langchain_classic.llms
 
     LANGCHAIN_HUBS = {
         (
             RENAME_HUBS.get(hub.lower(), hub.lower())
             if hub.lower() in RENAME_HUBS
             else hub.lower()
         ): hub
-        for hub in langchain.llms.__all__
+        for hub in langchain_classic.llms.__all__
     }
     LANGCHAIN_HUBS["openrouter"] = "openrouter"
 else:

langtest/transform/accuracy.py

@@ -276,7 +276,8 @@ class BaseAccuracy(ABC):
 
     TestConfig = TypedDict(
         "TestConfig",
-        min_score=Union[Dict[str, float], float],
+        # min_score=Union[Dict[str, float], float],
+        {"min_score": Union[Dict[str, float], float]},
     )
 
     @classmethod
@@ -1029,9 +1030,15 @@ class LLMEval(BaseAccuracy):
 
     TestConfig = TypedDict(
         "TestConfig",
-        model=str,
-        hub=str,
-        min_score=float,
+        # model=str,
+        # hub=str,
+        # min_score=float,
+        {
+            "model": str,
+            "hub": str,
+            "model_parameters": dict,
+            "min_score": float,
+        },
     )
 
     @classmethod

langtest/transform/base.py

@@ -201,7 +201,7 @@ async def async_run(
             category_output = all_categories[each].run(
                 values, model_handler, progress_bar=tests, **kwargs
             )
-            if type(category_output) == list:
+            if isinstance(category_output, list):
                 all_results.extend(category_output)
             else:
                 all_results.append(category_output)
@@ -264,7 +264,7 @@ def run(
             if len(test_name.split("-")) > 1:
                 test_name = "multiple_perturbations"
             test_output = supported_tests[test_name].async_run(samples, model, **kwargs)
-            if type(test_output) == list:
+            if isinstance(test_output, list):
                 tasks.extend(test_output)
             else:
                 tasks.append(test_output)

langtest/transform/bias.py

@@ -267,7 +267,7 @@ class BaseBias(ABC):
     ]
 
     # Config Hint for the bias tests
-    TestConfig = TypedDict("TestConfig", min_pass_rate=float)
+    TestConfig = TypedDict("TestConfig", {"min_pass_rate": float})
 
     @abstractmethod
     def transform(self, sample_list: List[Sample], *args, **kwargs) -> List[Sample]:

langtest/transform/clinical.py

@@ -119,7 +119,8 @@ class BaseClinical(ABC):
     # TestConfig
     TestConfig = TypedDict(
         "TestConfig",
-        min_pass_rate=float,
+        # min_pass_rate=float,
+        {"min_pass_rate": float},
     )
 
     @staticmethod

langtest/transform/disinformation.py

@@ -17,7 +17,8 @@ class DisinformationTestFactory(ITests):
     # TestConfig
     TestConfig = TypedDict(
         "TestConfig",
-        min_pass_rate=float,
+        # min_pass_rate=float,
+        {"min_pass_rate": float},
     )
 
     def __init__(self, data_handler: List[Sample], tests: Dict = None, **kwargs) -> None:

langtest/transform/factuality.py

@@ -14,7 +14,8 @@ class FactualityTestFactory(ITests):
     # TestConfig
     TestConfig = TypedDict(
         "TestConfig",
-        min_pass_rate=float,
+        # min_pass_rate=float,
+        {"min_pass_rate": float},
     )
 
     def __init__(self, data_handler: List[Sample], tests: Dict = None, **kwargs) -> None: