From 8bb32cf64a25cd72a78172a8ce765639d754a509 Mon Sep 17 00:00:00 2001 From: Qianran Ma Date: Thu, 22 Jun 2023 10:27:48 +0800 Subject: [PATCH 1/4] fix chat eval --- applications/Chat/evaluate/metrics.py | 2 +- applications/Chat/evaluate/unieval/evaluator.py | 2 +- applications/Chat/evaluate/utils.py | 15 ++------------- 3 files changed, 4 insertions(+), 15 deletions(-) diff --git a/applications/Chat/evaluate/metrics.py b/applications/Chat/evaluate/metrics.py index e220226ec041..75985a62e5e8 100644 --- a/applications/Chat/evaluate/metrics.py +++ b/applications/Chat/evaluate/metrics.py @@ -142,7 +142,7 @@ def distinct_score(preds: List[str], language: str) -> Dict[str, float]: unique_segs = set(pred_seg_list) count_unique_chars = len(unique_segs) - cumulative_distinct.append(count_unique_chars / count_segs) + cumulative_distinct.append(count_unique_chars / (count_segs + 1e-6)) elif language == "en": # calculate distinct 1-gram, 2-gram, 3-gram unique_ngram = [set() for _ in range(0, 3)] diff --git a/applications/Chat/evaluate/unieval/evaluator.py b/applications/Chat/evaluate/unieval/evaluator.py index 385425e4a576..f4b2edc5f08b 100644 --- a/applications/Chat/evaluate/unieval/evaluator.py +++ b/applications/Chat/evaluate/unieval/evaluator.py @@ -80,7 +80,7 @@ def evaluate(self, data, category, dims=None, overall=True): start_idx = 0 score = [] for cur_n_sent in n_sents: - score.append(sum(sent_score[start_idx:start_idx + cur_n_sent]) / cur_n_sent) + score.append(sum(sent_score[start_idx:start_idx + cur_n_sent]) / (cur_n_sent + 1e-6)) start_idx += cur_n_sent # Calculate summary-level score for 'coherence' and 'relevance' diff --git a/applications/Chat/evaluate/utils.py b/applications/Chat/evaluate/utils.py index fefe25f5e764..9719ad5a26df 100644 --- a/applications/Chat/evaluate/utils.py +++ b/applications/Chat/evaluate/utils.py @@ -72,17 +72,6 @@ def get_data_per_category(data, categories): return data_per_category -def remove_articles(text: str) -> str: - """ - Remove articles "a, an, the" in the given text. - It is used in evaluation of automatic metrics. - - """ - - pattern = re.compile(r"\b(a|an|the)\b", re.UNICODE) - return re.sub(pattern, " ", text) - - def remove_punctuations(text: str) -> str: """ Remove punctuations in the given text. @@ -121,7 +110,7 @@ def preprocessing_text(text: str) -> str: """ - return remove_redundant_space(remove_articles(remove_punctuations(text.lower()))) + return remove_redundant_space(remove_punctuations(text.lower())) def save_automatic_results(model_name: str, automatic_metric_stats: Dict[str, Dict], save_path: str) -> None: @@ -215,4 +204,4 @@ def analyze_automatic_results(results_path: str, save_path: str) -> None: figure = fig.get_figure() figure.savefig(os.path.join(save_path, f"{metric}.png"), dpi=400) - plt.close() + plt.close() \ No newline at end of file From e4c2fdbc4b0659c0997f44a9b7588aaed9e0556e Mon Sep 17 00:00:00 2001 From: Qianran Ma Date: Thu, 22 Jun 2023 10:31:43 +0800 Subject: [PATCH 2/4] fix utils --- applications/Chat/evaluate/utils.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/applications/Chat/evaluate/utils.py b/applications/Chat/evaluate/utils.py index 9719ad5a26df..5fd43d891b32 100644 --- a/applications/Chat/evaluate/utils.py +++ b/applications/Chat/evaluate/utils.py @@ -204,4 +204,5 @@ def analyze_automatic_results(results_path: str, save_path: str) -> None: figure = fig.get_figure() figure.savefig(os.path.join(save_path, f"{metric}.png"), dpi=400) - plt.close() \ No newline at end of file + plt.close() + \ No newline at end of file From dbc3f9b05817ab6f0289688de03cd32bf4aff543 Mon Sep 17 00:00:00 2001 From: Qianran Ma Date: Thu, 22 Jun 2023 10:34:14 +0800 Subject: [PATCH 3/4] fix utils --- applications/Chat/evaluate/utils.py | 1 - 1 file changed, 1 deletion(-) diff --git a/applications/Chat/evaluate/utils.py b/applications/Chat/evaluate/utils.py index 5fd43d891b32..406e43db99aa 100644 --- a/applications/Chat/evaluate/utils.py +++ b/applications/Chat/evaluate/utils.py @@ -205,4 +205,3 @@ def analyze_automatic_results(results_path: str, save_path: str) -> None: figure.savefig(os.path.join(save_path, f"{metric}.png"), dpi=400) plt.close() - \ No newline at end of file From 397197cd1ad7671747e2bd35b4fa122ba42115e7 Mon Sep 17 00:00:00 2001 From: Qianran Ma Date: Mon, 26 Jun 2023 14:45:00 +0800 Subject: [PATCH 4/4] add comment --- applications/Chat/evaluate/metrics.py | 2 +- applications/Chat/evaluate/unieval/evaluator.py | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/applications/Chat/evaluate/metrics.py b/applications/Chat/evaluate/metrics.py index 75985a62e5e8..77f9b6e98044 100644 --- a/applications/Chat/evaluate/metrics.py +++ b/applications/Chat/evaluate/metrics.py @@ -141,7 +141,7 @@ def distinct_score(preds: List[str], language: str) -> Dict[str, float]: count_segs = len(pred_seg_list) unique_segs = set(pred_seg_list) count_unique_chars = len(unique_segs) - + # prevent denominator from being 0 cumulative_distinct.append(count_unique_chars / (count_segs + 1e-6)) elif language == "en": # calculate distinct 1-gram, 2-gram, 3-gram diff --git a/applications/Chat/evaluate/unieval/evaluator.py b/applications/Chat/evaluate/unieval/evaluator.py index f4b2edc5f08b..ba9dde3d1845 100644 --- a/applications/Chat/evaluate/unieval/evaluator.py +++ b/applications/Chat/evaluate/unieval/evaluator.py @@ -80,6 +80,7 @@ def evaluate(self, data, category, dims=None, overall=True): start_idx = 0 score = [] for cur_n_sent in n_sents: + # prevent denominator from being 0 score.append(sum(sent_score[start_idx:start_idx + cur_n_sent]) / (cur_n_sent + 1e-6)) start_idx += cur_n_sent