Skip to content

Improve unit test coverage #700

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
ArshaanNazir merged 7 commits into from
Aug 14, 2023
Merged

Improve unit test coverage #700

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
7 commits
Select commit Hold shift + click to select a range
d6156c6
test(test_robustness): setup new test class
RakshitKhajuria Aug 8, 2023
4a0971a
test(test_robustness.py): added unittest
Prikshit7766 Aug 8, 2023
c93d409
refacto(accuracy.py): Added checks
RakshitKhajuria Aug 8, 2023
0c11e16
Merge branch 'improve-unit-test-coverage' of https://github.com/JohnS…
RakshitKhajuria Aug 8, 2023
38f8251
refacto accuracy.py
Prikshit7766 Aug 8, 2023
81e589f
pytest: test_accuracy.py
RakshitKhajuria Aug 8, 2023
2ef2e7a
test(test_accuracy.py):pytest for accuracy
Prikshit7766 Aug 8, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 2 additions & 2 deletions langtest/transform/__init__.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1068,9 +1068,9 @@ def transform(self) -> List[Sample]:
)

y_true = y_true.dropna()
params["test_name"] = test_name

transformed_samples = self.supported_tests[test_name].transform(
y_true, params
test_name, y_true, params
)

for sample in transformed_samples:
Expand Down
105 changes: 77 additions & 28 deletions langtest/transform/accuracy.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,7 @@ class BaseAccuracy(ABC):
alias_name = None
supported_tasks = ["ner", "text-classification"]

@staticmethod
@classmethod
@abstractmethod
def transform(y_true: List[Any], params: Dict) -> List[MinScoreSample]:
"""Abstract method that implements the accuracy measure.
Expand Down Expand Up @@ -77,10 +77,12 @@ class MinPrecisionScore(BaseAccuracy):
transform(y_true, y_pred) -> Any: Creates accuracy test results.
"""

alias_name = "min_precision_score"
alias_name = ["min_precision_score"]

@staticmethod
def transform(y_true: List[Any], params: Dict) -> List[MinScoreSample]:
@classmethod
def transform(
cls, test: str, y_true: List[Any], params: Dict
) -> List[MinScoreSample]:
"""Computes the minimum precision score for the given data.

Args:
Expand All @@ -90,6 +92,10 @@ def transform(y_true: List[Any], params: Dict) -> List[MinScoreSample]:
Returns:
List[MinScoreSample]: Precision test results.
"""
assert (
test in cls.alias_name
), f"Parameter 'test' should be in: {cls.alias_name}, got '{test}'"

labels = set(y_true) # .union(set(y_pred))

if isinstance(params["min_score"], dict):
Expand Down Expand Up @@ -149,10 +155,12 @@ class MinRecallScore(BaseAccuracy):
transform(y_true, y_pred) -> Any: Creates accuracy test results.
"""

alias_name = "min_recall_score"
alias_name = ["min_recall_score"]

@staticmethod
def transform(y_true: List[Any], params: Dict) -> List[MinScoreSample]:
@classmethod
def transform(
cls, test: str, y_true: List[Any], params: Dict
) -> List[MinScoreSample]:
"""Computes the minimum recall score for the given data.

Args:
Expand All @@ -162,6 +170,10 @@ def transform(y_true: List[Any], params: Dict) -> List[MinScoreSample]:
Returns:
List[MinScoreSample]: minimum recall results.
"""
assert (
test in cls.alias_name
), f"Parameter 'test' should be in: {cls.alias_name}, got '{test}'"

labels = set(y_true) # .union(set(y_pred))

if isinstance(params["min_score"], dict):
Expand Down Expand Up @@ -221,10 +233,12 @@ class MinF1Score(BaseAccuracy):

"""

alias_name = "min_f1_score"
alias_name = ["min_f1_score"]

@staticmethod
def transform(y_true: List[Any], params: Dict) -> List[MinScoreSample]:
@classmethod
def transform(
cls, test: str, y_true: List[Any], params: Dict
) -> List[MinScoreSample]:
"""Computes the minimum F1 score for the given data.

Args:
Expand All @@ -234,6 +248,10 @@ def transform(y_true: List[Any], params: Dict) -> List[MinScoreSample]:
Returns:
List[MinScoreSample]: F1 score test results.
"""
assert (
test in cls.alias_name
), f"Parameter 'test' should be in: {cls.alias_name}, got '{test}'"

labels = set(y_true)

if isinstance(params["min_score"], dict):
Expand Down Expand Up @@ -292,10 +310,12 @@ class MinMicroF1Score(BaseAccuracy):
alias_name (str): The name for config.
"""

alias_name = "min_micro_f1_score"
alias_name = ["min_micro_f1_score"]

@staticmethod
def transform(y_true: List[Any], params: Dict) -> List[MinScoreSample]:
@classmethod
def transform(
cls, test: str, y_true: List[Any], params: Dict
) -> List[MinScoreSample]:
"""Computes the minimum micro F1 score for the given data.

Args:
Expand All @@ -305,6 +325,10 @@ def transform(y_true: List[Any], params: Dict) -> List[MinScoreSample]:
Returns:
List[MinScoreSample]: The transformed data based on the minimum micro F1 score.
"""
assert (
test in cls.alias_name
), f"Parameter 'test' should be in: {cls.alias_name}, got '{test}'"

min_score = params["min_score"]

sample = MinScoreSample(
Expand Down Expand Up @@ -351,10 +375,12 @@ class MinMacroF1Score(BaseAccuracy):
transform(y_true, params) -> Any: Creates accuracy test results.
"""

alias_name = "min_macro_f1_score"
alias_name = ["min_macro_f1_score"]

@staticmethod
def transform(y_true: List[Any], params: Dict) -> List[MinScoreSample]:
@classmethod
def transform(
cls, test: str, y_true: List[Any], params: Dict
) -> List[MinScoreSample]:
"""Computes the minimum macro F1 score for the given data.

Args:
Expand All @@ -364,6 +390,9 @@ def transform(y_true: List[Any], params: Dict) -> List[MinScoreSample]:
Returns:
List[MinScoreSample]: The transformed data based on the minimum macro F1 score.
"""
assert (
test in cls.alias_name
), f"Parameter 'test' should be in: {cls.alias_name}, got '{test}'"
min_score = params["min_score"]

sample = MinScoreSample(
Expand Down Expand Up @@ -409,10 +438,12 @@ class MinWeightedF1Score(BaseAccuracy):
transform(y_true, params) -> Any: Creates accuracy test results.
"""

alias_name = "min_weighted_f1_score"
alias_name = ["min_weighted_f1_score"]

@staticmethod
def transform(y_true: List[Any], params: Dict) -> List[MinScoreSample]:
@classmethod
def transform(
cls, test: str, y_true: List[Any], params: Dict
) -> List[MinScoreSample]:
"""Computes the minimum weighted F1 score for the given data.

Args:
Expand All @@ -422,6 +453,9 @@ def transform(y_true: List[Any], params: Dict) -> List[MinScoreSample]:
Returns:
List[MinScoreSample]: The transformed data based on the minimum F1 score.
"""
assert (
test in cls.alias_name
), f"Parameter 'test' should be in: {cls.alias_name}, got '{test}'"
min_score = params["min_score"]

sample = MinScoreSample(
Expand Down Expand Up @@ -466,11 +500,13 @@ class MinEMcore(BaseAccuracy):
transform(y_true, y_pred) -> Any: Creates accuracy test results.
"""

alias_name = "min_exact_match_score"
alias_name = ["min_exact_match_score"]
supported_tasks = ["question-answering", "summarization"]

@staticmethod
def transform(y_true: List[Any], params: Dict) -> List[MinScoreSample]:
@classmethod
def transform(
cls, test: str, y_true: List[Any], params: Dict
) -> List[MinScoreSample]:
"""Computes the minimum F1 score for the given data.

Args:
Expand All @@ -480,6 +516,9 @@ def transform(y_true: List[Any], params: Dict) -> List[MinScoreSample]:
Returns:
List[MinScoreSample]: The transformed data based on the minimum F1 score.
"""
assert (
test in cls.alias_name
), f"Parameter 'test' should be in: {cls.alias_name}, got '{test}'"
min_score = params["min_score"]

sample = MinScoreSample(
Expand Down Expand Up @@ -528,11 +567,13 @@ class MinBLEUcore(BaseAccuracy):
transform(y_true, y_pred) -> Any: Creates accuracy test results.
"""

alias_name = "min_bleu_score"
alias_name = ["min_bleu_score"]
supported_tasks = ["question-answering", "summarization"]

@staticmethod
def transform(y_true: List[Any], params: Dict) -> List[MinScoreSample]:
@classmethod
def transform(
cls, test: str, y_true: List[Any], params: Dict
) -> List[MinScoreSample]:
"""Computes the minimum F1 score for the given data.

Args:
Expand All @@ -542,6 +583,9 @@ def transform(y_true: List[Any], params: Dict) -> List[MinScoreSample]:
Returns:
List[MinScoreSample]: The transformed data based on the minimum F1 score.
"""
assert (
test in cls.alias_name
), f"Parameter 'test' should be in: {cls.alias_name}, got '{test}'"
min_score = params["min_score"]

sample = MinScoreSample(
Expand Down Expand Up @@ -598,8 +642,10 @@ class MinROUGEcore(BaseAccuracy):
]
supported_tasks = ["question-answering", "summarization"]

@staticmethod
def transform(y_true: List[Any], params: Dict) -> List[MinScoreSample]:
@classmethod
def transform(
cls, test: str, y_true: List[Any], params: Dict
) -> List[MinScoreSample]:
"""Computes the minimum F1 score for the given data.

Args:
Expand All @@ -610,11 +656,14 @@ def transform(y_true: List[Any], params: Dict) -> List[MinScoreSample]:
Returns:
List[MinScoreSample]: The transformed data based on the minimum F1 score.
"""
assert (
test in cls.alias_name
), f"Parameter 'test' should be in: {cls.alias_name}, got '{test}'"
min_score = params["min_score"]

sample = MinScoreSample(
category="accuracy",
test_type=params["test_name"],
test_type=test,
expected_results=MinScoreOutput(min_score=min_score),
)

Expand Down
Loading