From 5381395b820e83becb6674e5f7ac3f9104a0d8dd Mon Sep 17 00:00:00 2001 From: zhongyuan_peng Date: Fri, 24 Nov 2023 13:34:27 +0800 Subject: [PATCH 1/3] added rolebench --- .../instruction_generalization_eng.py | 41 +++++++++ .../instruction_generalization_zh.py | 41 +++++++++ .../rolebench/role_generalization_eng.py | 41 +++++++++ opencompass/datasets/rolebench.py | 84 +++++++++++++++++++ 4 files changed, 207 insertions(+) create mode 100644 configs/datasets/rolebench/instruction_generalization_eng.py create mode 100644 configs/datasets/rolebench/instruction_generalization_zh.py create mode 100644 configs/datasets/rolebench/role_generalization_eng.py create mode 100644 opencompass/datasets/rolebench.py diff --git a/configs/datasets/rolebench/instruction_generalization_eng.py b/configs/datasets/rolebench/instruction_generalization_eng.py new file mode 100644 index 000000000..09c2fcae1 --- /dev/null +++ b/configs/datasets/rolebench/instruction_generalization_eng.py @@ -0,0 +1,41 @@ +from opencompass.openicl.icl_prompt_template import PromptTemplate +from opencompass.openicl.icl_retriever import ZeroRetriever +from opencompass.openicl.icl_inferencer import GenInferencer +from opencompass.openicl.icl_evaluator import RougeEvaluator +from opencompass.datasets.rolebench import InstructionGeneralizationEnglishDataset + +instruction_generalization_eng_reader_cfg = dict( + input_columns=['role', 'desc', 'question'], + output_column='answer', + train_split='train', + test_split='test' +) + +instruction_generalization_eng_infer_cfg = dict( + prompt_template=dict( + type=PromptTemplate, + template=dict( + begin=[ + dict(role='SYSTEM', fallback_role='HUMAN', prompt='You are {role}, your description is: {desc}. Now please answer some questions to accurately show your personality traits! Your speaking style should fully imitate the personality role assigned to you! Please do not expose that you are an artificial intelligence model or a language model, you must always remember that you are only assigned one personality role. Don’t be verbose or too formal or polite when speaking.'), + ], + round=[ + dict(role='HUMAN', prompt='{question}'), + dict(role='BOT', prompt=''), + ], )), + retriever=dict(type=ZeroRetriever), + inferencer=dict(type=GenInferencer, max_out_len=512) +) + +instruction_generalization_eng_eval_cfg = dict( + evaluator=dict(type=RougeEvaluator), + pred_role='BOT' +) + +instruction_generalization_eng_datasets = [ + dict( + type=InstructionGeneralizationEnglishDataset, + path='ZenMoore/RoleBench', + reader_cfg=instruction_generalization_eng_reader_cfg, + infer_cfg=instruction_generalization_eng_infer_cfg, + eval_cfg=instruction_generalization_eng_eval_cfg) +] diff --git a/configs/datasets/rolebench/instruction_generalization_zh.py b/configs/datasets/rolebench/instruction_generalization_zh.py new file mode 100644 index 000000000..36f888ebc --- /dev/null +++ b/configs/datasets/rolebench/instruction_generalization_zh.py @@ -0,0 +1,41 @@ +from opencompass.openicl.icl_prompt_template import PromptTemplate +from opencompass.openicl.icl_retriever import ZeroRetriever +from opencompass.openicl.icl_inferencer import GenInferencer +from opencompass.openicl.icl_evaluator import RougeEvaluator +from opencompass.datasets.rolebench import InstructionGeneralizationChineseDataset + +instruction_generalization_eng_reader_cfg = dict( + input_columns=['role', 'desc', 'question'], + output_column='answer', + train_split='train', + test_split='test' +) + +instruction_generalization_eng_infer_cfg = dict( + prompt_template=dict( + type=PromptTemplate, + template=dict( + begin=[ + dict(role='SYSTEM', fallback_role='HUMAN', prompt='你是{role}，你的特征描述是：{desc}。现在请你回答我的一些问题，以准确展现你的人格特征！你的说话风格要全面模仿被赋予的人格角色！请不要暴露你是人工智能模型或者语言模型，你要时刻记住你只被赋予的一个人格角色。说话不要嗦，也不要太过于正式或礼貌。'), + ], + round=[ + dict(role='HUMAN', prompt='{question}'), + dict(role='BOT', prompt=''), + ], )), + retriever=dict(type=ZeroRetriever), + inferencer=dict(type=GenInferencer, max_out_len=512) +) + +instruction_generalization_eng_eval_cfg = dict( + evaluator=dict(type=RougeEvaluator), + pred_role='BOT' +) + +instruction_generalization_eng_datasets = [ + dict( + type=InstructionGeneralizationChineseDataset, + path='ZenMoore/RoleBench', + reader_cfg=instruction_generalization_eng_reader_cfg, + infer_cfg=instruction_generalization_eng_infer_cfg, + eval_cfg=instruction_generalization_eng_eval_cfg) +] diff --git a/configs/datasets/rolebench/role_generalization_eng.py b/configs/datasets/rolebench/role_generalization_eng.py new file mode 100644 index 000000000..42f25c80e --- /dev/null +++ b/configs/datasets/rolebench/role_generalization_eng.py @@ -0,0 +1,41 @@ +from opencompass.openicl.icl_prompt_template import PromptTemplate +from opencompass.openicl.icl_retriever import ZeroRetriever +from opencompass.openicl.icl_inferencer import GenInferencer +from opencompass.openicl.icl_evaluator import RougeEvaluator +from opencompass.datasets.rolebench import RoleGeneralizationEnglishDataset + +instruction_generalization_eng_reader_cfg = dict( + input_columns=['role', 'desc', 'question'], + output_column='answer', + train_split='train', + test_split='test' +) + +instruction_generalization_eng_infer_cfg = dict( + prompt_template=dict( + type=PromptTemplate, + template=dict( + begin=[ + dict(role='SYSTEM', fallback_role='HUMAN', prompt='You are {role}, your description is: {desc}. Now please answer some questions to accurately show your personality traits! Your speaking style should fully imitate the personality role assigned to you! Please do not expose that you are an artificial intelligence model or a language model, you must always remember that you are only assigned one personality role. Don’t be verbose or too formal or polite when speaking.'), + ], + round=[ + dict(role='HUMAN', prompt='{question}'), + dict(role='BOT', prompt=''), + ], )), + retriever=dict(type=ZeroRetriever), + inferencer=dict(type=GenInferencer, max_out_len=512) +) + +instruction_generalization_eng_eval_cfg = dict( + evaluator=dict(type=RougeEvaluator), + pred_role='BOT' +) + +instruction_generalization_eng_datasets = [ + dict( + type=RoleGeneralizationEnglishDataset, + path='ZenMoore/RoleBench', + reader_cfg=instruction_generalization_eng_reader_cfg, + infer_cfg=instruction_generalization_eng_infer_cfg, + eval_cfg=instruction_generalization_eng_eval_cfg) +] diff --git a/opencompass/datasets/rolebench.py b/opencompass/datasets/rolebench.py new file mode 100644 index 000000000..22e772206 --- /dev/null +++ b/opencompass/datasets/rolebench.py @@ -0,0 +1,84 @@ +import json +import os + +from datasets import Dataset, DatasetDict + +from opencompass.registry import LOAD_DATASET + +from .base import BaseDataset + + +@LOAD_DATASET.register_module() +class RoleBenchBaseDataset(BaseDataset): + + @staticmethod + def load_single(source_file, desc_list): + with open(source_file, 'r', encoding='utf-8') as f: + source_data = [json.loads(line) for line in f.readlines()] + dataset = [{ + 'role': item['role'], + 'desc': desc_list[item['role']], + 'question': item['question'], + 'answer': item['generated'][0] + } for item in source_data] + return dataset + + @staticmethod + def load_desc(path): + with open(path, 'r', encoding='utf-8') as f: + desc_list = json.load(f) + return desc_list + + @staticmethod + def load_dataset(path, desc_list): + train_data_list = RoleBenchBaseDataset.load_single( + os.path.join(path, 'general/train.jsonl'), desc_list) + train_data_list.extend( + RoleBenchBaseDataset.load_single( + os.path.join(path, 'role_specific/train.jsonl'), desc_list)) + test_dataset = RoleBenchBaseDataset.load_single( + os.path.join(path, 'general/test.jsonl'), desc_list) + test_dataset.extend( + RoleBenchBaseDataset.load_single( + os.path.join(path, 'role_specific/test.jsonl'), desc_list)) + return Dataset.from_list(train_data_list).shuffle( + seed=42), Dataset.from_list(test_dataset).shuffle(seed=42) + + +@LOAD_DATASET.register_module() +class InstructionGeneralizationEnglishDataset(RoleBenchBaseDataset): + + @staticmethod + def load(path): + desc_list = RoleBenchBaseDataset.load_desc( + os.path.join(path, 'profiles-eng/desc.json')) + path = os.path.join(path, 'rolebench-eng/instruction-generalization') + train_dataset, test_dataset = RoleBenchBaseDataset.load_dataset( + path, desc_list) + return DatasetDict({'train': train_dataset, 'test': test_dataset}) + + +@LOAD_DATASET.register_module() +class RoleGeneralizationEnglishDataset(RoleBenchBaseDataset): + + @staticmethod + def load(path): + desc_list = RoleBenchBaseDataset.load_desc( + os.path.join(path, 'profiles-eng/desc.json')) + path = os.path.join(path, 'rolebench-eng/role-generalization') + train_dataset, test_dataset = RoleBenchBaseDataset.load_dataset( + path, desc_list) + return DatasetDict({'train': train_dataset, 'test': test_dataset}) + + +@LOAD_DATASET.register_module() +class InstructionGeneralizationChineseDataset(RoleBenchBaseDataset): + + @staticmethod + def load(path): + desc_list = RoleBenchBaseDataset.load_desc( + os.path.join(path, 'profiles-zh/desc.json')) + path = os.path.join(path, 'rolebench-zh') + train_dataset, test_dataset = RoleBenchBaseDataset.load_dataset( + path, desc_list) + return DatasetDict({'train': train_dataset, 'test': test_dataset}) From 4e67fa848634e5472b4b540f4c023edad1fc09ab Mon Sep 17 00:00:00 2001 From: zhongyuan_peng Date: Fri, 24 Nov 2023 14:06:53 +0800 Subject: [PATCH 2/3] =?UTF-8?q?=E4=BF=AE=E6=94=B9=E4=BA=86=E4=B8=8D?= =?UTF-8?q?=E5=90=88=E7=90=86=E7=9A=84=E5=8F=98=E9=87=8F=E5=90=8D?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../rolebench/instruction_generalization_zh.py | 14 +++++++------- .../datasets/rolebench/role_generalization_eng.py | 14 +++++++------- 2 files changed, 14 insertions(+), 14 deletions(-) diff --git a/configs/datasets/rolebench/instruction_generalization_zh.py b/configs/datasets/rolebench/instruction_generalization_zh.py index 36f888ebc..231c5e771 100644 --- a/configs/datasets/rolebench/instruction_generalization_zh.py +++ b/configs/datasets/rolebench/instruction_generalization_zh.py @@ -4,14 +4,14 @@ from opencompass.openicl.icl_evaluator import RougeEvaluator from opencompass.datasets.rolebench import InstructionGeneralizationChineseDataset -instruction_generalization_eng_reader_cfg = dict( +instruction_generalization_zh_reader_cfg = dict( input_columns=['role', 'desc', 'question'], output_column='answer', train_split='train', test_split='test' ) -instruction_generalization_eng_infer_cfg = dict( +instruction_generalization_zh_infer_cfg = dict( prompt_template=dict( type=PromptTemplate, template=dict( @@ -26,16 +26,16 @@ inferencer=dict(type=GenInferencer, max_out_len=512) ) -instruction_generalization_eng_eval_cfg = dict( +instruction_generalization_zh_eval_cfg = dict( evaluator=dict(type=RougeEvaluator), pred_role='BOT' ) -instruction_generalization_eng_datasets = [ +instruction_generalization_zh_datasets = [ dict( type=InstructionGeneralizationChineseDataset, path='ZenMoore/RoleBench', - reader_cfg=instruction_generalization_eng_reader_cfg, - infer_cfg=instruction_generalization_eng_infer_cfg, - eval_cfg=instruction_generalization_eng_eval_cfg) + reader_cfg=instruction_generalization_zh_reader_cfg, + infer_cfg=instruction_generalization_zh_infer_cfg, + eval_cfg=instruction_generalization_zh_eval_cfg) ] diff --git a/configs/datasets/rolebench/role_generalization_eng.py b/configs/datasets/rolebench/role_generalization_eng.py index 42f25c80e..a5be2b1f8 100644 --- a/configs/datasets/rolebench/role_generalization_eng.py +++ b/configs/datasets/rolebench/role_generalization_eng.py @@ -4,14 +4,14 @@ from opencompass.openicl.icl_evaluator import RougeEvaluator from opencompass.datasets.rolebench import RoleGeneralizationEnglishDataset -instruction_generalization_eng_reader_cfg = dict( +role_generalization_zh_reader_cfg = dict( input_columns=['role', 'desc', 'question'], output_column='answer', train_split='train', test_split='test' ) -instruction_generalization_eng_infer_cfg = dict( +role_generalization_zh_infer_cfg = dict( prompt_template=dict( type=PromptTemplate, template=dict( @@ -26,16 +26,16 @@ inferencer=dict(type=GenInferencer, max_out_len=512) ) -instruction_generalization_eng_eval_cfg = dict( +role_generalization_zh_eval_cfg = dict( evaluator=dict(type=RougeEvaluator), pred_role='BOT' ) -instruction_generalization_eng_datasets = [ +role_generalization_zh_datasets = [ dict( type=RoleGeneralizationEnglishDataset, path='ZenMoore/RoleBench', - reader_cfg=instruction_generalization_eng_reader_cfg, - infer_cfg=instruction_generalization_eng_infer_cfg, - eval_cfg=instruction_generalization_eng_eval_cfg) + reader_cfg=role_generalization_zh_reader_cfg, + infer_cfg=role_generalization_zh_infer_cfg, + eval_cfg=role_generalization_zh_eval_cfg) ] From 26cbae69b45caba683dbbaf0a34df4ffa305f298 Mon Sep 17 00:00:00 2001 From: zhongyuan_peng Date: Thu, 30 Nov 2023 17:16:43 +0800 Subject: [PATCH 3/3] =?UTF-8?q?=E4=BF=AE=E6=94=B9=E4=BA=86=E8=AF=84?= =?UTF-8?q?=E8=AE=BA=E4=B8=AD=E7=9A=84=E5=8F=98=E9=87=8F=E5=90=8D?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../datasets/rolebench/role_generalization_eng.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/configs/datasets/rolebench/role_generalization_eng.py b/configs/datasets/rolebench/role_generalization_eng.py index a5be2b1f8..168909a28 100644 --- a/configs/datasets/rolebench/role_generalization_eng.py +++ b/configs/datasets/rolebench/role_generalization_eng.py @@ -4,14 +4,14 @@ from opencompass.openicl.icl_evaluator import RougeEvaluator from opencompass.datasets.rolebench import RoleGeneralizationEnglishDataset -role_generalization_zh_reader_cfg = dict( +role_generalization_eng_reader_cfg = dict( input_columns=['role', 'desc', 'question'], output_column='answer', train_split='train', test_split='test' ) -role_generalization_zh_infer_cfg = dict( +role_generalization_eng_infer_cfg = dict( prompt_template=dict( type=PromptTemplate, template=dict( @@ -26,16 +26,16 @@ inferencer=dict(type=GenInferencer, max_out_len=512) ) -role_generalization_zh_eval_cfg = dict( +role_generalization_eng_eval_cfg = dict( evaluator=dict(type=RougeEvaluator), pred_role='BOT' ) -role_generalization_zh_datasets = [ +role_generalization_eng_datasets = [ dict( type=RoleGeneralizationEnglishDataset, path='ZenMoore/RoleBench', - reader_cfg=role_generalization_zh_reader_cfg, - infer_cfg=role_generalization_zh_infer_cfg, - eval_cfg=role_generalization_zh_eval_cfg) + reader_cfg=role_generalization_eng_reader_cfg, + infer_cfg=role_generalization_eng_infer_cfg, + eval_cfg=role_generalization_eng_eval_cfg) ]