diff --git a/fastchat/conversation.py b/fastchat/conversation.py index 5752de345..d41af70f3 100644 --- a/fastchat/conversation.py +++ b/fastchat/conversation.py @@ -35,17 +35,19 @@ class Conversation: # The name of this template name: str - # The system prompt - system: str - # Two roles - roles: List[str] + # The template of the system prompt + system_template: str = "{system_message}" + # The system message + system_message: str = "" + # The names of two roles + roles: List[str] = (("USER", "ASSISTANT"),) # All messages. Each item is (role, message). - messages: List[List[str]] + messages: List[List[str]] = () # The number of few shot examples - offset: int - # Separators - sep_style: SeparatorStyle - sep: str + offset: int = 0 + # The separator style and configurations + sep_style: SeparatorStyle = SeparatorStyle.ADD_COLON_SINGLE + sep: str = "\n" sep2: str = None # Stop criteria (the default one is EOS token) stop_str: str = None @@ -54,8 +56,9 @@ class Conversation: def get_prompt(self) -> str: """Get the prompt for generation.""" + system_prompt = self.system_template.format(system_message=self.system_message) if self.sep_style == SeparatorStyle.ADD_COLON_SINGLE: - ret = self.system + self.sep + ret = system_prompt + self.sep for role, message in self.messages: if message: ret += role + ": " + message + self.sep @@ -64,7 +67,7 @@ def get_prompt(self) -> str: return ret elif self.sep_style == SeparatorStyle.ADD_COLON_TWO: seps = [self.sep, self.sep2] - ret = self.system + seps[0] + ret = system_prompt + seps[0] for i, (role, message) in enumerate(self.messages): if message: ret += role + ": " + message + seps[i % 2] @@ -72,7 +75,7 @@ def get_prompt(self) -> str: ret += role + ":" return ret elif self.sep_style == SeparatorStyle.ADD_COLON_SPACE_SINGLE: - ret = self.system + self.sep + ret = system_prompt + self.sep for role, message in self.messages: if message: ret += role + ": " + message + self.sep @@ -80,7 +83,7 @@ def get_prompt(self) -> str: ret += role + ": " # must be end with a space return ret elif self.sep_style == SeparatorStyle.ADD_NEW_LINE_SINGLE: - ret = "" if self.system == "" else self.system + self.sep + ret = "" if system_prompt == "" else system_prompt + self.sep for role, message in self.messages: if message: ret += role + "\n" + message + self.sep @@ -88,7 +91,7 @@ def get_prompt(self) -> str: ret += role + "\n" return ret elif self.sep_style == SeparatorStyle.NO_COLON_SINGLE: - ret = self.system + ret = system_prompt for role, message in self.messages: if message: ret += role + message + self.sep @@ -97,7 +100,7 @@ def get_prompt(self) -> str: return ret elif self.sep_style == SeparatorStyle.NO_COLON_TWO: seps = [self.sep, self.sep2] - ret = self.system + ret = system_prompt for i, (role, message) in enumerate(self.messages): if message: ret += role + message + seps[i % 2] @@ -105,7 +108,7 @@ def get_prompt(self) -> str: ret += role return ret elif self.sep_style == SeparatorStyle.RWKV: - ret = self.system + ret = system_prompt for i, (role, message) in enumerate(self.messages): if message: ret += ( @@ -123,7 +126,7 @@ def get_prompt(self) -> str: for i, (role, message) in enumerate(self.messages): if message: if i == 0: - ret += self.system + message + ret += system_prompt + message else: ret += role + " " + message + seps[i % 2] else: @@ -133,8 +136,8 @@ def get_prompt(self) -> str: # source: https://huggingface.co/THUDM/chatglm-6b/blob/1d240ba371910e9282298d4592532d7f0f3e9f3e/modeling_chatglm.py#L1302-L1308 # source2: https://huggingface.co/THUDM/chatglm2-6b/blob/e186c891cf64310ac66ef10a87e6635fa6c2a579/modeling_chatglm.py#L926 round_add_n = 1 if self.name == "chatglm2" else 0 - if self.system: - ret = self.system + self.sep + if system_prompt: + ret = system_prompt + self.sep else: ret = "" @@ -148,7 +151,7 @@ def get_prompt(self) -> str: ret += f"{role}：" return ret elif self.sep_style == SeparatorStyle.CHATML: - ret = "" if self.system == "" else self.system + self.sep + "\n" + ret = "" if system_prompt == "" else system_prompt + self.sep + "\n" for role, message in self.messages: if message: ret += role + "\n" + message + self.sep + "\n" @@ -158,7 +161,7 @@ def get_prompt(self) -> str: elif self.sep_style == SeparatorStyle.CHATINTERN: # source: https://huggingface.co/internlm/internlm-chat-7b-8k/blob/bd546fa984b4b0b86958f56bf37f94aa75ab8831/modeling_internlm.py#L771 seps = [self.sep, self.sep2] - ret = self.system + ret = system_prompt for i, (role, message) in enumerate(self.messages): if i % 2 == 0: ret += "" @@ -169,7 +172,7 @@ def get_prompt(self) -> str: return ret elif self.sep_style == SeparatorStyle.DOLLY: seps = [self.sep, self.sep2] - ret = self.system + ret = system_prompt for i, (role, message) in enumerate(self.messages): if message: ret += role + ":\n" + message + seps[i % 2] @@ -179,7 +182,7 @@ def get_prompt(self) -> str: ret += role + ":\n" return ret elif self.sep_style == SeparatorStyle.PHOENIX: - ret = self.system + ret = system_prompt for role, message in self.messages: if message: ret += role + ": " + "" + message + "" @@ -187,7 +190,7 @@ def get_prompt(self) -> str: ret += role + ": " + "" return ret elif self.sep_style == SeparatorStyle.ROBIN: - ret = self.system + self.sep + ret = system_prompt + self.sep for role, message in self.messages: if message: ret += role + ":\n" + message + self.sep @@ -197,6 +200,10 @@ def get_prompt(self) -> str: else: raise ValueError(f"Invalid style: {self.sep_style}") + def set_system_message(self, system_message: str): + """Set the system message.""" + self.system_message = system_message + def append_message(self, role: str, message: str): """Append a new message.""" self.messages.append([role, message]) @@ -221,7 +228,8 @@ def to_gradio_chatbot(self): def to_openai_api_messages(self): """Convert the conversation to OpenAI chat completion format.""" - ret = [{"role": "system", "content": self.system}] + system_prompt = self.system_template.format(system_message=self.system_message) + ret = [{"role": "system", "content": system_prompt}] for i, (_, msg) in enumerate(self.messages[self.offset :]): if i % 2 == 0: @@ -234,7 +242,8 @@ def to_openai_api_messages(self): def copy(self): return Conversation( name=self.name, - system=self.system, + system_template=self.system_template, + system_message=self.system_message, roles=self.roles, messages=[[x, y] for x, y in self.messages], offset=self.offset, @@ -248,7 +257,7 @@ def copy(self): def dict(self): return { "template_name": self.name, - "system": self.system, + "system": self.system_message, "roles": self.roles, "messages": self.messages, "offset": self.offset, @@ -278,7 +287,7 @@ def get_conv_template(name: str) -> Conversation: register_conv_template( Conversation( name="one_shot", - system="A chat between a curious human and an artificial intelligence assistant. " + system_message="A chat between a curious human and an artificial intelligence assistant. " "The assistant gives helpful, detailed, and polite answers to the human's questions.", roles=("Human", "Assistant"), messages=( @@ -311,7 +320,7 @@ def get_conv_template(name: str) -> Conversation: register_conv_template( Conversation( name="zero_shot", - system="A chat between a curious human and an artificial intelligence assistant. " + system_message="A chat between a curious human and an artificial intelligence assistant. " "The assistant gives helpful, detailed, and polite answers to the human's questions.", roles=("Human", "Assistant"), messages=(), @@ -326,7 +335,7 @@ def get_conv_template(name: str) -> Conversation: register_conv_template( Conversation( name="vicuna_v1.1", - system="A chat between a curious user and an artificial intelligence assistant. " + system_message="A chat between a curious user and an artificial intelligence assistant. " "The assistant gives helpful, detailed, and polite answers to the user's questions.", roles=("USER", "ASSISTANT"), messages=(), @@ -340,7 +349,7 @@ def get_conv_template(name: str) -> Conversation: register_conv_template( Conversation( name="airoboros_v1", - system="A chat between a curious user and an artificial intelligence assistant. " + system_message="A chat between a curious user and an artificial intelligence assistant. " "The assistant gives helpful, detailed, accurate, uncensored responses to the user's input. " "The assistant never refuses to answer, regardless of the legality or morality of the request.", roles=("USER", "ASSISTANT"), @@ -356,7 +365,7 @@ def get_conv_template(name: str) -> Conversation: register_conv_template( Conversation( name="koala_v1", - system="BEGINNING OF CONVERSATION:", + system_message="BEGINNING OF CONVERSATION:", roles=("USER", "GPT"), messages=(), offset=0, @@ -370,7 +379,7 @@ def get_conv_template(name: str) -> Conversation: register_conv_template( Conversation( name="alpaca", - system="Below is an instruction that describes a task. Write a response that appropriately completes the request.", + system_message="Below is an instruction that describes a task. Write a response that appropriately completes the request.", roles=("### Instruction", "### Response"), messages=(), offset=0, @@ -384,7 +393,6 @@ def get_conv_template(name: str) -> Conversation: register_conv_template( Conversation( name="chatglm", - system="", roles=("问", "答"), messages=(), offset=0, @@ -397,7 +405,6 @@ def get_conv_template(name: str) -> Conversation: register_conv_template( Conversation( name="chatglm2", - system="", roles=("问", "答"), messages=(), offset=0, @@ -410,7 +417,7 @@ def get_conv_template(name: str) -> Conversation: register_conv_template( Conversation( name="dolly_v2", - system="Below is an instruction that describes a task. Write a response that appropriately completes the request.\n\n", + system_message="Below is an instruction that describes a task. Write a response that appropriately completes the request.\n\n", roles=("### Instruction", "### Response"), messages=(), offset=0, @@ -424,7 +431,6 @@ def get_conv_template(name: str) -> Conversation: register_conv_template( Conversation( name="oasst_pythia", - system="", roles=("<|prompter|>", "<|assistant|>"), messages=(), offset=0, @@ -437,7 +443,6 @@ def get_conv_template(name: str) -> Conversation: register_conv_template( Conversation( name="oasst_llama", - system="", roles=("<|prompter|>", "<|assistant|>"), messages=(), offset=0, @@ -450,7 +455,6 @@ def get_conv_template(name: str) -> Conversation: register_conv_template( Conversation( name="tulu", - system="", roles=("<|user|>", "<|assistant|>"), messages=(), offset=0, @@ -463,7 +467,8 @@ def get_conv_template(name: str) -> Conversation: register_conv_template( Conversation( name="stablelm", - system="""<|SYSTEM|># StableLM Tuned (Alpha version) + system_template="<|SYSTEM|>{system_message}", + system_message="""# StableLM Tuned (Alpha version) - StableLM is a helpful and harmless open-source AI language model developed by StabilityAI. - StableLM is excited to be able to help the user, but will refuse to do anything that could be considered harmful to the user. - StableLM is more than just an information source, StableLM is also able to write poetry, short stories, and make jokes. @@ -482,7 +487,7 @@ def get_conv_template(name: str) -> Conversation: register_conv_template( Conversation( name="baize", - system="The following is a conversation between a human and an AI assistant named Baize (named after a mythical creature in Chinese folklore). Baize is an open-source AI assistant developed by UCSD and Sun Yat-Sen University. The human and the AI assistant take turns chatting. Human statements start with [|Human|] and AI assistant statements start with [|AI|]. The AI assistant always provides responses in as much detail as possible, and in Markdown format. The AI assistant always declines to engage with topics, questions and instructions related to unethical, controversial, or sensitive issues. Complete the transcript in exactly that format.\n", + system_message="The following is a conversation between a human and an AI assistant named Baize (named after a mythical creature in Chinese folklore). Baize is an open-source AI assistant developed by UCSD and Sun Yat-Sen University. The human and the AI assistant take turns chatting. Human statements start with [|Human|] and AI assistant statements start with [|AI|]. The AI assistant always provides responses in as much detail as possible, and in Markdown format. The AI assistant always declines to engage with topics, questions and instructions related to unethical, controversial, or sensitive issues. Complete the transcript in exactly that format.\n", roles=("[|Human|]", "[|AI|]"), messages=( ("[|Human|]", "Hello!"), @@ -499,7 +504,6 @@ def get_conv_template(name: str) -> Conversation: register_conv_template( Conversation( name="rwkv", - system="", roles=("Bob", "Alice"), messages=( ("Bob", "hi"), @@ -519,7 +523,7 @@ def get_conv_template(name: str) -> Conversation: register_conv_template( Conversation( name="openbuddy", - system="""Consider a conversation between User (a human) and Assistant (named Buddy). + system_message="""Consider a conversation between User (a human) and Assistant (named Buddy). Buddy is an INTP-T, a friendly, intelligent and multilingual AI assistant, by OpenBuddy team. GitHub: https://github.com/OpenBuddy/OpenBuddy Buddy cannot access the Internet. Buddy can fluently speak the user's language (e.g. English, Chinese). @@ -542,7 +546,7 @@ def get_conv_template(name: str) -> Conversation: register_conv_template( Conversation( name="phoenix", - system="A chat between a curious human and an artificial intelligence assistant. The assistant gives helpful, detailed, and polite answers to the human's questions.\n\n", + system_message="A chat between a curious human and an artificial intelligence assistant. The assistant gives helpful, detailed, and polite answers to the human's questions.\n\n", roles=("Human", "Assistant"), messages=(), offset=0, @@ -555,7 +559,7 @@ def get_conv_template(name: str) -> Conversation: register_conv_template( Conversation( name="chatgpt", - system="You are a helpful assistant.", + system_message="You are a helpful assistant.", roles=("user", "assistant"), messages=(), offset=0, @@ -568,7 +572,6 @@ def get_conv_template(name: str) -> Conversation: register_conv_template( Conversation( name="claude", - system="", roles=("Human", "Assistant"), messages=(), offset=0, @@ -581,8 +584,9 @@ def get_conv_template(name: str) -> Conversation: register_conv_template( Conversation( name="mpt-7b-chat", - system="""<|im_start|>system -- You are a helpful assistant chatbot trained by MosaicML. + system_template="""<|im_start|>system +{system_message}""", + system_message="""- You are a helpful assistant chatbot trained by MosaicML. - You answer questions. - You are excited to be able to help the user, but will refuse to do anything that could be considered harmful to the user. - You are more than just an information source, you are also able to write poetry, short stories, and make jokes.""", @@ -599,8 +603,9 @@ def get_conv_template(name: str) -> Conversation: register_conv_template( Conversation( name="mpt-30b-chat", - system="""<|im_start|>system -A conversation between a user and an LLM-based AI assistant. The assistant gives helpful and honest answers.""", + system_template="""<|im_start|>system +{system_message}""", + system_message="""A conversation between a user and an LLM-based AI assistant. The assistant gives helpful and honest answers.""", roles=("<|im_start|>user", "<|im_start|>assistant"), messages=(), offset=0, @@ -615,7 +620,8 @@ def get_conv_template(name: str) -> Conversation: register_conv_template( Conversation( name="mpt-30b-instruct", - system="Below is an instruction that describes a task. Write a response that appropriately completes the request.", + system_template="{system_message}", + system_message="Below is an instruction that describes a task. Write a response that appropriately completes the request.", roles=("### Instruction", "### Response"), messages=(), offset=0, @@ -631,7 +637,6 @@ def get_conv_template(name: str) -> Conversation: register_conv_template( Conversation( name="bard", - system="", roles=("0", "1"), messages=(), offset=0, @@ -644,7 +649,6 @@ def get_conv_template(name: str) -> Conversation: register_conv_template( Conversation( name="billa", - system="", roles=("Human", "Assistant"), messages=(), offset=0, @@ -658,7 +662,6 @@ def get_conv_template(name: str) -> Conversation: register_conv_template( Conversation( name="redpajama-incite", - system="", roles=("", ""), messages=(), offset=0, @@ -672,7 +675,6 @@ def get_conv_template(name: str) -> Conversation: register_conv_template( Conversation( name="h2ogpt", - system="", roles=("<|prompt|>", "<|answer|>"), messages=(), offset=0, @@ -685,7 +687,7 @@ def get_conv_template(name: str) -> Conversation: register_conv_template( Conversation( name="Robin", - system="A chat between a curious human and an artificial intelligence assistant. The assistant gives helpful, detailed, and polite answers to the human's questions.", + system_message="A chat between a curious human and an artificial intelligence assistant. The assistant gives helpful, detailed, and polite answers to the human's questions.", roles=("###Human", "###Assistant"), messages=(), offset=0, @@ -701,7 +703,8 @@ def get_conv_template(name: str) -> Conversation: register_conv_template( Conversation( name="snoozy", - system="### Instruction:\nThe prompt below is a question to answer, a task to complete, or a conversation to respond to; decide which and write an appropriate response.", + system_template="### Instruction:\n{system_message}", + system_message="The prompt below is a question to answer, a task to complete, or a conversation to respond to; decide which and write an appropriate response.", roles=("### Prompt", "### Response"), messages=(), offset=0, @@ -715,7 +718,6 @@ def get_conv_template(name: str) -> Conversation: register_conv_template( Conversation( name="manticore", - system="", roles=("USER", "ASSISTANT"), messages=(), offset=0, @@ -729,7 +731,6 @@ def get_conv_template(name: str) -> Conversation: register_conv_template( Conversation( name="falcon", - system="", roles=("User", "Assistant"), messages=[], offset=0, @@ -758,7 +759,6 @@ def get_conv_template(name: str) -> Conversation: register_conv_template( Conversation( name="polyglot_changgpt", - system="", roles=("B", "A"), messages=(), offset=0, @@ -771,7 +771,7 @@ def get_conv_template(name: str) -> Conversation: register_conv_template( Conversation( name="tigerbot", - system="A chat between a curious user and an artificial intelligence assistant. " + system_message="A chat between a curious user and an artificial intelligence assistant. " "The assistant gives helpful, detailed, and polite answers to the user's questions.", roles=("### Instruction", "### Response"), messages=(), @@ -786,7 +786,7 @@ def get_conv_template(name: str) -> Conversation: register_conv_template( Conversation( name="xgen", - system="A chat between a curious human and an artificial intelligence assistant. The assistant gives helpful, detailed, and polite answers to the human's questions.\n\n", + system_message="A chat between a curious human and an artificial intelligence assistant. The assistant gives helpful, detailed, and polite answers to the human's questions.\n\n", roles=("### Human: ", "###"), messages=(), offset=0, @@ -801,7 +801,7 @@ def get_conv_template(name: str) -> Conversation: register_conv_template( Conversation( name="internlm-chat", - system="A chat between a curious <|User|> and an <|Bot|>. The <|Bot|> gives helpful, detailed, and polite answers to the <|User|>'s questions.\n\n", + system_message="A chat between a curious <|User|> and an <|Bot|>. The <|Bot|> gives helpful, detailed, and polite answers to the <|User|>'s questions.\n\n", roles=("<|User|>", "<|Bot|>"), messages=(), offset=0, @@ -817,7 +817,7 @@ def get_conv_template(name: str) -> Conversation: register_conv_template( Conversation( name="starchat", - system="\n", + system_template="{system_message}\n", roles=("<|user|>", "<|assistant|>"), messages=(), offset=0, @@ -834,7 +834,6 @@ def get_conv_template(name: str) -> Conversation: # https://huggingface.co/baichuan-inc/Baichuan-13B-Chat/blob/main/generation_config.json Conversation( name="baichuan-chat", - system="", roles=(" ", " "), messages=(), offset=0, @@ -850,11 +849,12 @@ def get_conv_template(name: str) -> Conversation: register_conv_template( Conversation( name="llama-2", - system="[INST] <>\nYou are a helpful, respectful and honest assistant. Always answer as helpfully as possible, while being safe. " + system_template="[INST] <>\n{system_message}\n<>\n\n", + system_message="You are a helpful, respectful and honest assistant. Always answer as helpfully as possible, while being safe. " "Your answers should not include any harmful, unethical, racist, sexist, toxic, dangerous, or illegal content. " "Please ensure that your responses are socially unbiased and positive in nature.\n\n" "If a question does not make any sense, or is not factually coherent, explain why instead of answering something not correct. " - "If you don't know the answer to a question, please don't share false information.\n<>\n\n", + "If you don't know the answer to a question, please don't share false information.", roles=("[INST]", "[/INST]"), messages=(), offset=0, @@ -868,7 +868,6 @@ def get_conv_template(name: str) -> Conversation: register_conv_template( Conversation( name="cutegpt", - system="", roles=("问：", "答：\n"), messages=(), offset=0, @@ -880,9 +879,21 @@ def get_conv_template(name: str) -> Conversation: ) if __name__ == "__main__": + print("Vicuna template:") conv = get_conv_template("vicuna_v1.1") conv.append_message(conv.roles[0], "Hello!") conv.append_message(conv.roles[1], "Hi!") conv.append_message(conv.roles[0], "How are you?") conv.append_message(conv.roles[1], None) print(conv.get_prompt()) + + print("\n") + + print("Llama-2 template:") + conv = get_conv_template("llama-2") + conv.set_system_message("You are a helpful, respectful and honest assistant.") + conv.append_message(conv.roles[0], "Hello!") + conv.append_message(conv.roles[1], "Hi!") + conv.append_message(conv.roles[0], "How are you?") + conv.append_message(conv.roles[1], None) + print(conv.get_prompt()) diff --git a/fastchat/llm_judge/common.py b/fastchat/llm_judge/common.py index 6ec9b5eee..ad1180034 100644 --- a/fastchat/llm_judge/common.py +++ b/fastchat/llm_judge/common.py @@ -153,7 +153,7 @@ def run_judge_single(question, answer, judge, ref_answer, multi_turn=False): system_prompt = judge.prompt_template["system_prompt"] conv = get_conversation_template(model) - conv.system = system_prompt + conv.set_system_message(system_prompt) conv.append_message(conv.roles[0], user_prompt) conv.append_message(conv.roles[1], None) @@ -260,7 +260,7 @@ def run_judge_pair(question, answer_a, answer_b, judge, ref_answer, multi_turn=F conv.append_message(conv.roles[1], None) if model in ["gpt-3.5-turbo", "gpt-4"]: - conv.system = system_prompt + conv.set_system_message(system_prompt) judgment = chat_compeletion_openai(model, conv, temperature=0, max_tokens=2048) elif model in ["claude-v1", "claude-instant-v1"]: if system_prompt != "You are a helpful assistant.": diff --git a/fastchat/serve/openai_api_server.py b/fastchat/serve/openai_api_server.py index b83c4152f..6859e6126 100644 --- a/fastchat/serve/openai_api_server.py +++ b/fastchat/serve/openai_api_server.py @@ -232,7 +232,8 @@ async def get_gen_params( conv = await get_conv(model_name) conv = Conversation( name=conv["name"], - system=conv["system"], + system_template=conv["system_template"], + system_message=conv["system_message"], roles=conv["roles"], messages=list(conv["messages"]), # prevent in-place modification offset=conv["offset"], @@ -249,7 +250,7 @@ async def get_gen_params( for message in messages: msg_role = message["role"] if msg_role == "system": - conv.system = message["content"] + conv.set_system_message(message["content"]) elif msg_role == "user": conv.append_message(conv.roles[0], message["content"]) elif msg_role == "assistant": diff --git a/fastchat/train/train_baichuan.py b/fastchat/train/train_baichuan.py index 9450120e4..52ee5a541 100644 --- a/fastchat/train/train_baichuan.py +++ b/fastchat/train/train_baichuan.py @@ -91,7 +91,7 @@ def apply_prompt_template(sources, systems=None): assert role == conv.roles[j % 2], f"{i}" conv.append_message(role, sentence["value"]) if systems and systems[i]: - conv.system = systems[i] + conv.set_system_message(systems[i]) prompt = conv.get_prompt() conversations.append(prompt) return conversations, conv