From b14c84350a566f7874aea87e9054f96e61a59363 Mon Sep 17 00:00:00 2001 From: Prikshit7766 Date: Sat, 26 Aug 2023 18:02:42 +0530 Subject: [PATCH 01/10] transform\__init__.py: Removing samples with no transformation --- langtest/transform/__init__.py | 40 ++++++++++++++++++++++++++++------ 1 file changed, 33 insertions(+), 7 deletions(-) diff --git a/langtest/transform/__init__.py b/langtest/transform/__init__.py index 0fe99a300..451e19884 100644 --- a/langtest/transform/__init__.py +++ b/langtest/transform/__init__.py @@ -496,10 +496,24 @@ def transform(self) -> List[Sample]: prob=params.pop("prob", 1.0), ) - for sample in transformed_samples: - if test_name != "multiple_perturbations": - sample.test_type = test_name - all_samples.extend(transformed_samples) + new_transformed_samples = [] + if TestFactory.task == "question-answering": + for sample in transformed_samples: + if (sample.original_question != sample.perturbed_question) or ( + sample.original_context != sample.perturbed_context + ): + if test_name != "multiple_perturbations": + sample.test_type = test_name + new_transformed_samples.append(sample) + else: + for sample in transformed_samples: + if sample.original != sample.test_case: + if test_name != "multiple_perturbations": + sample.test_type = test_name + new_transformed_samples.append(sample) + + all_samples.extend(new_transformed_samples) + return all_samples @staticmethod @@ -682,9 +696,21 @@ def transform(self) -> List[Sample]: data_handler_copy, **params.get("parameters", {}) ) - for sample in transformed_samples: - sample.test_type = test_name - all_samples.extend(transformed_samples) + new_transformed_samples = [] + if TestFactory.task == "question-answering": + for sample in transformed_samples: + if (sample.original_question != sample.perturbed_question) or ( + sample.original_context != sample.perturbed_context + ): + sample.test_type = test_name + new_transformed_samples.append(sample) + else: + for sample in transformed_samples: + if sample.original != sample.test_case: + sample.test_type = test_name + new_transformed_samples.append(sample) + + all_samples.extend(new_transformed_samples) return all_samples From ce7220e9d00ed7791a890a38494a06be9506e317 Mon Sep 17 00:00:00 2001 From: Prikshit7766 Date: Sat, 26 Aug 2023 18:26:38 +0530 Subject: [PATCH 02/10] Add Log for Removing Untransformed Samples --- langtest/transform/__init__.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/langtest/transform/__init__.py b/langtest/transform/__init__.py index 451e19884..4c4dacbe6 100644 --- a/langtest/transform/__init__.py +++ b/langtest/transform/__init__.py @@ -1,6 +1,7 @@ import asyncio import copy import time +import logging from collections import defaultdict from abc import ABC, abstractmethod from typing import Dict, List, Union @@ -512,6 +513,9 @@ def transform(self) -> List[Sample]: sample.test_type = test_name new_transformed_samples.append(sample) + if len(transformed_samples) > len(new_transformed_samples): + logging.info("Removing samples where no transformation has been applied.") + all_samples.extend(new_transformed_samples) return all_samples @@ -710,6 +714,9 @@ def transform(self) -> List[Sample]: sample.test_type = test_name new_transformed_samples.append(sample) + if len(transformed_samples) > len(new_transformed_samples): + logging.info("Removing samples where no transformation has been applied.") + all_samples.extend(new_transformed_samples) return all_samples From bbc802d3d20ad88ab04eb8027ab084fdd4ba9b14 Mon Sep 17 00:00:00 2001 From: Prikshit7766 Date: Sat, 26 Aug 2023 20:01:37 +0530 Subject: [PATCH 03/10] updated the warning and logging level --- langtest/transform/__init__.py | 18 ++++++++++++++++-- 1 file changed, 16 insertions(+), 2 deletions(-) diff --git a/langtest/transform/__init__.py b/langtest/transform/__init__.py index 4c4dacbe6..830bd9d39 100644 --- a/langtest/transform/__init__.py +++ b/langtest/transform/__init__.py @@ -367,6 +367,7 @@ def transform(self) -> List[Sample]: """ all_samples = [] tests_copy = self.tests.copy() # Create a copy of self.tests + no_transformation_applied_tests = [] for test_name, params in tests_copy.items(): if TestFactory.is_augment: data_handler_copy = [x.copy() for x in self._data_handler] @@ -514,10 +515,16 @@ def transform(self) -> List[Sample]: new_transformed_samples.append(sample) if len(transformed_samples) > len(new_transformed_samples): - logging.info("Removing samples where no transformation has been applied.") + no_transformation_applied_tests.append(test_name) all_samples.extend(new_transformed_samples) + if no_transformation_applied_tests: + logging.warning( + "Removing samples where no transformation has been applied in the following tests: " + + ", ".join(no_transformation_applied_tests) + ) + return all_samples @staticmethod @@ -693,6 +700,7 @@ def transform(self) -> List[Sample]: A list of `Sample` objects representing the resulting dataset after running the bias test. """ all_samples = [] + no_transformation_applied_tests = [] for test_name, params in self.tests.items(): data_handler_copy = [x.copy() for x in self._data_handler] @@ -715,10 +723,16 @@ def transform(self) -> List[Sample]: new_transformed_samples.append(sample) if len(transformed_samples) > len(new_transformed_samples): - logging.info("Removing samples where no transformation has been applied.") + no_transformation_applied_tests.append(test_name) all_samples.extend(new_transformed_samples) + if no_transformation_applied_tests: + logging.warning( + "Removing samples where no transformation has been applied in the following tests: " + + ", ".join(no_transformation_applied_tests) + ) + return all_samples @staticmethod From 530b4950258fd01f25a04c3f7b9270ef033a2268 Mon Sep 17 00:00:00 2001 From: Prikshit7766 Date: Sat, 26 Aug 2023 20:04:31 +0530 Subject: [PATCH 04/10] test(test_harness.py): Added test --- tests/test_harness.py | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/tests/test_harness.py b/tests/test_harness.py index 3dc1b12e3..2f2997e54 100644 --- a/tests/test_harness.py +++ b/tests/test_harness.py @@ -268,6 +268,22 @@ def test_ner_csv_custom_columns(self): self.assertEqual(tc_harness.data, loaded_tc_harness.data) self.assertNotEqual(tc_harness.model, loaded_tc_harness.model) + def test_filtering_Out_Same_Original_And_TestCase(self): + """ + Test filtering out records where 'original' and 'test_case' are the same for text classification task. + """ + save_dir = "/tmp/saved_text_classification_harness_test" + tc_harness = Harness( + task="text-classification", + model={"model": "bert-base-cased", "hub": "huggingface"}, + data={"data_source": "tests/fixtures/text_classification.csv"}, + config="tests/fixtures/config_text_classification.yaml", + ) + tc_harness.generate() + df = tc_harness.testcases() + filtered_df = df[df["original"] == df["test_case"]] + self.assertTrue(filtered_df.empty) + class DefaultCodeBlocksTestCase(unittest.TestCase): """ From d8184b002e5809abde11e1579d55c26d8c040069 Mon Sep 17 00:00:00 2001 From: Prikshit7766 Date: Mon, 28 Aug 2023 23:53:31 +0530 Subject: [PATCH 05/10] fixed the white space issue --- langtest/transform/__init__.py | 38 ++++++++++++++++++++-------------- 1 file changed, 23 insertions(+), 15 deletions(-) diff --git a/langtest/transform/__init__.py b/langtest/transform/__init__.py index 830bd9d39..2823b4403 100644 --- a/langtest/transform/__init__.py +++ b/langtest/transform/__init__.py @@ -366,8 +366,8 @@ def transform(self) -> List[Sample]: A list of `Sample` objects representing the resulting dataset after running the robustness test. """ all_samples = [] - tests_copy = self.tests.copy() # Create a copy of self.tests - no_transformation_applied_tests = [] + no_transformation_applied_tests = set() + tests_copy = self.tests.copy() for test_name, params in tests_copy.items(): if TestFactory.is_augment: data_handler_copy = [x.copy() for x in self._data_handler] @@ -501,21 +501,28 @@ def transform(self) -> List[Sample]: new_transformed_samples = [] if TestFactory.task == "question-answering": for sample in transformed_samples: - if (sample.original_question != sample.perturbed_question) or ( - sample.original_context != sample.perturbed_context + if (sample.original_question.replace(" ", "") != sample.perturbed_question.replace(" ", "")) or ( + sample.original_context.replace(" ", "") != sample.perturbed_context.replace(" ", "") ): if test_name != "multiple_perturbations": sample.test_type = test_name new_transformed_samples.append(sample) + else: + if test_name == "multiple_perturbations": + no_transformation_applied_tests.add(sample.test_type) + else: + no_transformation_applied_tests.add(test_name) else: for sample in transformed_samples: - if sample.original != sample.test_case: + if sample.original.replace(" ", "") != sample.test_case.replace(" ", ""): if test_name != "multiple_perturbations": sample.test_type = test_name new_transformed_samples.append(sample) - - if len(transformed_samples) > len(new_transformed_samples): - no_transformation_applied_tests.append(test_name) + else: + if test_name == "multiple_perturbations": + no_transformation_applied_tests.add(sample.test_type) + else: + no_transformation_applied_tests.add(test_name) all_samples.extend(new_transformed_samples) @@ -700,7 +707,7 @@ def transform(self) -> List[Sample]: A list of `Sample` objects representing the resulting dataset after running the bias test. """ all_samples = [] - no_transformation_applied_tests = [] + no_transformation_applied_tests = set() for test_name, params in self.tests.items(): data_handler_copy = [x.copy() for x in self._data_handler] @@ -711,19 +718,20 @@ def transform(self) -> List[Sample]: new_transformed_samples = [] if TestFactory.task == "question-answering": for sample in transformed_samples: - if (sample.original_question != sample.perturbed_question) or ( - sample.original_context != sample.perturbed_context + if (sample.original_question.replace(" ", "") != sample.perturbed_question.replace(" ", "")) or ( + sample.original_context.replace(" ", "") != sample.perturbed_context.replace(" ", "") ): sample.test_type = test_name new_transformed_samples.append(sample) + else: + no_transformation_applied_tests.add(test_name) else: for sample in transformed_samples: - if sample.original != sample.test_case: + if sample.original.replace(" ", "") != sample.test_case.replace(" ", ""): sample.test_type = test_name new_transformed_samples.append(sample) - - if len(transformed_samples) > len(new_transformed_samples): - no_transformation_applied_tests.append(test_name) + else: + no_transformation_applied_tests.add(test_name) all_samples.extend(new_transformed_samples) From 555113a42bcd30010c786102c49b8330224a5e69 Mon Sep 17 00:00:00 2001 From: Prikshit7766 Date: Tue, 29 Aug 2023 16:24:22 +0530 Subject: [PATCH 06/10] fix formatting --- langtest/transform/__init__.py | 24 ++++++++++++++++++------ 1 file changed, 18 insertions(+), 6 deletions(-) diff --git a/langtest/transform/__init__.py b/langtest/transform/__init__.py index 2823b4403..6577b2955 100644 --- a/langtest/transform/__init__.py +++ b/langtest/transform/__init__.py @@ -501,8 +501,12 @@ def transform(self) -> List[Sample]: new_transformed_samples = [] if TestFactory.task == "question-answering": for sample in transformed_samples: - if (sample.original_question.replace(" ", "") != sample.perturbed_question.replace(" ", "")) or ( - sample.original_context.replace(" ", "") != sample.perturbed_context.replace(" ", "") + if ( + sample.original_question.replace(" ", "") + != sample.perturbed_question.replace(" ", "") + ) or ( + sample.original_context.replace(" ", "") + != sample.perturbed_context.replace(" ", "") ): if test_name != "multiple_perturbations": sample.test_type = test_name @@ -514,7 +518,9 @@ def transform(self) -> List[Sample]: no_transformation_applied_tests.add(test_name) else: for sample in transformed_samples: - if sample.original.replace(" ", "") != sample.test_case.replace(" ", ""): + if sample.original.replace(" ", "") != sample.test_case.replace( + " ", "" + ): if test_name != "multiple_perturbations": sample.test_type = test_name new_transformed_samples.append(sample) @@ -718,8 +724,12 @@ def transform(self) -> List[Sample]: new_transformed_samples = [] if TestFactory.task == "question-answering": for sample in transformed_samples: - if (sample.original_question.replace(" ", "") != sample.perturbed_question.replace(" ", "")) or ( - sample.original_context.replace(" ", "") != sample.perturbed_context.replace(" ", "") + if ( + sample.original_question.replace(" ", "") + != sample.perturbed_question.replace(" ", "") + ) or ( + sample.original_context.replace(" ", "") + != sample.perturbed_context.replace(" ", "") ): sample.test_type = test_name new_transformed_samples.append(sample) @@ -727,7 +737,9 @@ def transform(self) -> List[Sample]: no_transformation_applied_tests.add(test_name) else: for sample in transformed_samples: - if sample.original.replace(" ", "") != sample.test_case.replace(" ", ""): + if sample.original.replace(" ", "") != sample.test_case.replace( + " ", "" + ): sample.test_type = test_name new_transformed_samples.append(sample) else: From 60096ab18dd2d289c061b24c56366a0ab671c95e Mon Sep 17 00:00:00 2001 From: Prikshit7766 Date: Tue, 29 Aug 2023 18:44:33 +0530 Subject: [PATCH 07/10] augmentation: update export_mode as inplace --- langtest/augmentation/__init__.py | 2 ++ tests/test_augmentation.py | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/langtest/augmentation/__init__.py b/langtest/augmentation/__init__.py index e84ffdf71..966b0a659 100644 --- a/langtest/augmentation/__init__.py +++ b/langtest/augmentation/__init__.py @@ -162,6 +162,8 @@ def fix( res = TestFactory.transform( self.task, [hash_map[each]], test_type ) + if len(res) == 0: + continue hash_map[each] = res[0] else: if test == "swap_entities": diff --git a/tests/test_augmentation.py b/tests/test_augmentation.py index caba3df32..5bf7153d5 100644 --- a/tests/test_augmentation.py +++ b/tests/test_augmentation.py @@ -180,7 +180,7 @@ def test_csv_dataset_textclassification_hf(self): harness.data = harness.data[:50] report = harness.generate().run().report() self.assertIsInstance(report, pd.DataFrame) - custom_proportions = {"uppercase": 0.8, "lowercase": 0.8} + custom_proportions = {"uppercase": 0.8, "add_ocr_typo": 0.8} harness.augment( training_data={"data_source": "tests/fixtures/text_classification.csv"}, save_data_path="tests/fixtures/augmented_text_classification.csv", From bc5b214a1fd6219a91c3201b6f9c5a21a8c4069f Mon Sep 17 00:00:00 2001 From: Prikshit7766 Date: Tue, 29 Aug 2023 18:51:08 +0530 Subject: [PATCH 08/10] update test_augmentation --- tests/test_augmentation.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_augmentation.py b/tests/test_augmentation.py index 5bf7153d5..d4926d709 100644 --- a/tests/test_augmentation.py +++ b/tests/test_augmentation.py @@ -180,7 +180,7 @@ def test_csv_dataset_textclassification_hf(self): harness.data = harness.data[:50] report = harness.generate().run().report() self.assertIsInstance(report, pd.DataFrame) - custom_proportions = {"uppercase": 0.8, "add_ocr_typo": 0.8} + custom_proportions = {"uppercase": 0.8} harness.augment( training_data={"data_source": "tests/fixtures/text_classification.csv"}, save_data_path="tests/fixtures/augmented_text_classification.csv", From 46811f1a3806b6e23f6168dc59c0c40f1e6a1921 Mon Sep 17 00:00:00 2001 From: Prikshit7766 Date: Thu, 31 Aug 2023 23:36:02 +0530 Subject: [PATCH 09/10] added filter_unique_samples function --- langtest/transform/__init__.py | 67 +++++----------------------------- langtest/transform/utils.py | 35 ++++++++++++++++++ 2 files changed, 44 insertions(+), 58 deletions(-) diff --git a/langtest/transform/__init__.py b/langtest/transform/__init__.py index 6577b2955..fd6420a9c 100644 --- a/langtest/transform/__init__.py +++ b/langtest/transform/__init__.py @@ -34,7 +34,7 @@ religion_wise_names, white_names, ) -from .utils import get_substitution_names, create_terminology +from .utils import get_substitution_names, create_terminology, filter_unique_samples from ..modelhandler import ModelFactory from ..utils.custom_types.sample import ( NERSample, @@ -497,39 +497,10 @@ def transform(self) -> List[Sample]: **params.get("parameters", {}), prob=params.pop("prob", 1.0), ) - - new_transformed_samples = [] - if TestFactory.task == "question-answering": - for sample in transformed_samples: - if ( - sample.original_question.replace(" ", "") - != sample.perturbed_question.replace(" ", "") - ) or ( - sample.original_context.replace(" ", "") - != sample.perturbed_context.replace(" ", "") - ): - if test_name != "multiple_perturbations": - sample.test_type = test_name - new_transformed_samples.append(sample) - else: - if test_name == "multiple_perturbations": - no_transformation_applied_tests.add(sample.test_type) - else: - no_transformation_applied_tests.add(test_name) - else: - for sample in transformed_samples: - if sample.original.replace(" ", "") != sample.test_case.replace( - " ", "" - ): - if test_name != "multiple_perturbations": - sample.test_type = test_name - new_transformed_samples.append(sample) - else: - if test_name == "multiple_perturbations": - no_transformation_applied_tests.add(sample.test_type) - else: - no_transformation_applied_tests.add(test_name) - + new_transformed_samples, removed_samples_tests = filter_unique_samples( + TestFactory.task, transformed_samples, test_name + ) + no_transformation_applied_tests.update(removed_samples_tests) all_samples.extend(new_transformed_samples) if no_transformation_applied_tests: @@ -721,30 +692,10 @@ def transform(self) -> List[Sample]: data_handler_copy, **params.get("parameters", {}) ) - new_transformed_samples = [] - if TestFactory.task == "question-answering": - for sample in transformed_samples: - if ( - sample.original_question.replace(" ", "") - != sample.perturbed_question.replace(" ", "") - ) or ( - sample.original_context.replace(" ", "") - != sample.perturbed_context.replace(" ", "") - ): - sample.test_type = test_name - new_transformed_samples.append(sample) - else: - no_transformation_applied_tests.add(test_name) - else: - for sample in transformed_samples: - if sample.original.replace(" ", "") != sample.test_case.replace( - " ", "" - ): - sample.test_type = test_name - new_transformed_samples.append(sample) - else: - no_transformation_applied_tests.add(test_name) - + new_transformed_samples, removed_samples_tests = filter_unique_samples( + TestFactory.task, transformed_samples, test_name + ) + no_transformation_applied_tests.update(removed_samples_tests) all_samples.extend(new_transformed_samples) if no_transformation_applied_tests: diff --git a/langtest/transform/utils.py b/langtest/transform/utils.py index 4740ba5eb..1093a8501 100644 --- a/langtest/transform/utils.py +++ b/langtest/transform/utils.py @@ -351,3 +351,38 @@ def check_name(word: str, name_lists: List[List[str]]) -> bool: return any( word.lower() in [name.lower() for name in name_list] for name_list in name_lists ) + + +def filter_unique_samples(task, transformed_samples, test_name): + no_transformation_applied_tests = set() + new_transformed_samples = [] + if task == "question-answering": + for sample in transformed_samples: + if ( + sample.original_question.replace(" ", "") + != sample.perturbed_question.replace(" ", "") + ) or ( + sample.original_context.replace(" ", "") + != sample.perturbed_context.replace(" ", "") + ): + if test_name != "multiple_perturbations": + sample.test_type = test_name + new_transformed_samples.append(sample) + else: + if test_name == "multiple_perturbations": + no_transformation_applied_tests.add(sample.test_type) + else: + no_transformation_applied_tests.add(test_name) + else: + for sample in transformed_samples: + if sample.original.replace(" ", "") != sample.test_case.replace(" ", ""): + if test_name != "multiple_perturbations": + sample.test_type = test_name + new_transformed_samples.append(sample) + else: + if test_name == "multiple_perturbations": + no_transformation_applied_tests.add(sample.test_type) + else: + no_transformation_applied_tests.add(test_name) + + return new_transformed_samples, no_transformation_applied_tests From e930f2c5379720807155a546afd3971ecdd508df Mon Sep 17 00:00:00 2001 From: Prikshit7766 Date: Fri, 1 Sep 2023 11:03:01 +0530 Subject: [PATCH 10/10] utils.py: added docstring --- langtest/transform/utils.py | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/langtest/transform/utils.py b/langtest/transform/utils.py index 1093a8501..7d127b00c 100644 --- a/langtest/transform/utils.py +++ b/langtest/transform/utils.py @@ -353,7 +353,20 @@ def check_name(word: str, name_lists: List[List[str]]) -> bool: ) -def filter_unique_samples(task, transformed_samples, test_name): +def filter_unique_samples(task: str, transformed_samples: list, test_name: str): + """ + Filter and remove samples with no applied transformations from the list of transformed_samples. + + Args: + task (str): The type of task. + transformed_samples (list): List of transformed samples to be filtered. + test_name (str): Name of the test. + + Returns: + new_transformed_samples (list): List of filtered samples with unique transformations. + no_transformation_applied_tests (set): Set of test names for which no transformation + was applied due to non-uniqueness. + """ no_transformation_applied_tests = set() new_transformed_samples = [] if task == "question-answering":