💥 [Fix] slow tokenizer issue, and finish by stop_sequences

messagers/message_composer.py

@@ -150,9 +150,12 @@ class MessageComposer:
             self.merged_str = "<bos>" + "\n".join(self.merged_str_list)
         # https://huggingface.co/NousResearch/Nous-Hermes-2-Mixtral-8x7B-DPO#prompt-format
         # https://huggingface.co/openchat/openchat-3.5-0106
-        # elif self.model in ["openchat-3.5", "nous-mixtral-8x7b"]:
-        elif self.model in ["openchat-3.5", "command-r-plus", "gemma-7b"]:
-            tokenizer = AutoTokenizer.from_pretrained(self.model_fullname)
+        # https://huggingface.co/01-ai/Yi-1.5-34B-Chat
+        elif self.model in ["openchat-3.5", "command-r-plus", "gemma-7b", "yi-1.5-34b"]:
+            # https://discuss.huggingface.co/t/error-with-new-tokenizers-urgent/2847/5
+            tokenizer = AutoTokenizer.from_pretrained(
+                self.model_fullname, use_fast=False
+            )
             self.merged_str = tokenizer.apply_chat_template(
                 messages, tokenize=False, add_generation_prompt=True
             )