Upload 9 files

.gitattributes

@@ -33,3 +33,4 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+model.data filter=lfs diff=lfs merge=lfs -text

genai_config.json

@@ -0,0 +1,56 @@
+{
+    "model": {
+        "bos_token_id": 0,
+        "context_length": 8192,
+        "decoder": {
+            "session_options": {
+                "log_id": "onnxruntime-genai",
+                "custom_ops_library": "onnxruntime_vitis_ai_custom_ops.dll",
+                "provider_options": [
+                    {
+                        "VitisAI": {
+                            "config_file": ".\\libs\\vaip_llm.json"
+                        }
+                    }
+                ]
+            },
+            "filename": "model.onnx",
+            "head_size": 128,
+            "hidden_size": 4096,
+            "inputs": {
+                "input_ids": "input_ids",
+                "attention_mask": "attention_mask",
+                "past_key_names": "past_key_values.%d.key",
+                "past_value_names": "past_key_values.%d.value"
+            },
+            "outputs": {
+                "logits": "logits",
+                "present_key_names": "present.%d.key",
+                "present_value_names": "present.%d.value"
+            },
+            "num_attention_heads": 32,
+            "num_hidden_layers": 28,
+            "num_key_value_heads": 2
+        },
+        "eos_token_id": 2,
+        "pad_token_id": 0,
+        "type": "chatglm",
+        "vocab_size": 65024
+    },
+    "search": {
+        "diversity_penalty": 0.0,
+        "do_sample": false,
+        "early_stopping": true,
+        "length_penalty": 1.0,
+        "max_length": 8192,
+        "min_length": 0,
+        "no_repeat_ngram_size": 0,
+        "num_beams": 1,
+        "num_return_sequences": 1,
+        "past_present_share_buffer": true,
+        "repetition_penalty": 1.0,
+        "temperature": 1.0,
+        "top_k": 1,
+        "top_p": 1.0
+    }
+}

model.data

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a8e60c0f2bd5314b3b51269d293f6a0f7775020cf6e9c1d8bbf057c48e10ff64
+size 3791874048

model.onnx

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:67931c5992f4721670bc6f73491da788c977f2546dedb76697d497d6cecd54a7
+size 213850

prompts.txt

@@ -0,0 +1,2 @@
+2048------------------------------
+In recent years, artificial intelligence (AI) has revolutionized numerous industries, with healthcare being one of the most promising fields. The integration of AI in healthcare systems has the potential to transform patient care, diagnostics, and treatment plans. Imagine a world where AI-powered algorithms can predict diseases before they manifest, provide personalized treatment plans based on genetic information, and even assist in complex surgeries with unparalleled precision. One of the most significant advantages of AI in healthcare is its ability to analyze vast amounts of data quickly and accurately. Traditional methods of data analysis in healthcare often involve manual processes that are time-consuming and prone to human error. AI, on the other hand, can sift through millions of patient records, medical images, and research papers in a fraction of the time it would take a human. This ability to process and analyze big data allows for more accurate diagnoses and more effective treatment plans. For example, consider the case of a patient presenting with symptoms that could indicate several different conditions. An AI system could analyze the patientΓÇÖs medical history, compare it with millions of other cases, and suggest the most likely diagnosis. It could also recommend a personalized treatment plan based on the patientΓÇÖs genetic makeup, lifestyle, and other factors. This level of precision medicine has the potential to improve patient outcomes significantly. However, the implementation of AI in healthcare is not without its challenges. One of the main concerns is the ethical implications of using AI in such a sensitive field. For instance, who is responsible if an AI system makes a wrong diagnosis? How do we ensure that AI systems are not biased in their decision-making processes? Moreover, there are concerns about patient privacy and the security of sensitive medical data. Another challenge is the integration of AI systems into existing healthcare infrastructure. Many healthcare systems are already burdened with outdated technology and limited resources. Integrating advanced AI systems into these environments requires significant investment and training. Additionally, there is the question of how AI will impact the roles of healthcare professionals. While AI can assist doctors and nurses in their work, there is concern that it could also lead to job displacement. Despite these challenges, the potential benefits of AI in healthcare are immense. For example, AI-powered robots are already being used in some hospitals to assist with surgeries. These robots can perform delicate procedures with a level of precision that is difficult for humans to achieve. AI is also being used to develop new drugs and treatment plans. By analyzing the molecular structure of diseases, AI can help researchers identify potential treatments faster than traditional methods. Moreover, AI has In recent years, artificial intelligence (AI) has revolutionized numerous industries, with healthcare being one of the most promising fields. The integration of AI in healthcare systems has the potential to transform patient care, diagnostics, and treatment plans. Imagine a world where AI-powered algorithms can predict diseases before they manifest, provide personalized treatment plans based on genetic information, and even assist in complex surgeries with unparalleled precision. One of the most significant advantages of AI in healthcare is its ability to analyze vast amounts of data quickly and accurately. Traditional methods of data analysis in healthcare often involve manual processes that are time-consuming and prone to human error. AI, on the other hand, can sift through millions of patient records, medical images, and research papers in a fraction of the time it would take a human. This ability to process and analyze big data allows for more accurate diagnoses and more effective treatment plans. For example, consider the case of a patient presenting with symptoms that could indicate several different conditions. An AI system could analyze the patientΓÇÖs medical history, compare it with millions of other cases, and suggest the most likely diagnosis. It could also recommend a personalized treatment plan based on the patientΓÇÖs genetic makeup, lifestyle, and other factors. This level of precision medicine has the potential to improve patient outcomes significantly. However, the implementation of AI in healthcare is not without its challenges. One of the main concerns is the ethical implications of using AI in such a sensitive field. For instance, who is responsible if an AI system makes a wrong diagnosis? How do we ensure that AI systems are not biased in their decision-making processes? Moreover, there are concerns about patient privacy and the security of sensitive medical data. Another challenge is the integration of AI systems into existing healthcare infrastructure. Many healthcare systems are already burdened with outdated technology and limited resources. Integrating advanced AI systems into these environments requires significant investment and training. Additionally, there is the question of how AI will impact the roles of healthcare professionals. While AI can assist doctors and nurses in their work, there is concern that it could also lead to job displacement. Despite these challenges, the potential benefits of AI in healthcare are immense. For example, AI-powered robots are already being used in some hospitals to assist with surgeries. These robots can perform delicate procedures with a level of precision that is difficult for humans to achieve. AI is also being used to develop new drugs and treatment plans. By analyzing the molecular structure of diseases, AI can help researchers identify potential treatments faster than traditional methods. Moreover, AI has In recent years, artificial intelligence (AI) has revolutionized numerous industries, with healthcare being one of the most promising fields. The integration of AI in healthcare systems has the potential to transform patient care, diagnostics, and treatment plans. Imagine a world where AI-powered algorithms can predict diseases before they manifest, provide personalized treatment plans based on genetic information, and even assist in complex surgeries with unparalleled precision. One of the most significant advantages of AI in healthcare is its ability to analyze vast amounts of data quickly and accurately. Traditional methods of data analysis in healthcare often involve manual processes that are time-consuming and prone to human error. AI, on the other hand, can sift through millions of patient records, medical images, and research papers in a fraction of the time it would take a human. This ability to process and analyze big data allows for more accurate diagnoses and more effective treatment plans. For example, consider the case of a patient presenting with symptoms that could indicate several different conditions. An AI system could analyze the patientΓÇÖs medical history, compare it with millions of other cases, and suggest the most likely diagnosis. It could also recommend a personalized treatment plan based on the patientΓÇÖs genetic makeup, lifestyle, and other factors. This level of precision medicine has the potential to improve patient outcomes significantly. However, the implementation of AI in healthcare is not without its challenges. One of the main concerns is the ethical implications of using AI in such a sensitive field. For instance, who is responsible if an AI system makes a wrong diagnosis? How do we ensure that AI systems are not biased in their decision-making processes? Moreover, there are concerns about patient privacy and the security of sensitive medical data. Another challenge is the integration of AI systems into existing healthcare infrastructure. Many healthcare systems are already burdened with outdated technology and limited resources. Integrating advanced AI systems into these environments requires significant investment and training. Additionally, there is the question of how AI will impact the roles of healthcare professionals. While AI can assist doctors and nurses in their work, there is concern that it could also lead to job displacement. Despite these challenges, the potential benefits of AI in healthcare are immense. For example, AI-powered robots are already being used in some hospitals to assist with surgeries. These robots can perform delicate procedures with a level of precision that is difficult for humans to achieve. AI is also being used to develop new drugs and treatment plans. By analyzing the molecular structure of diseases, AI can help researchers identify potential treatments faster than traditional methods. Moreover, AI has In recent years, artificial intelligence (AI) has revolutionized numerous industries, with healthcare being one of the most promising fields. The integration of AI in healthcare systems has the potential to transform patient care, diagnostics, and treatment plans. Imagine a world where AI-powered algorithms can predict diseases before they manifest, provide personalized treatment plans based on genetic information, and even assist in complex surgeries with unparalleled precision. One of the most significant advantages of AI in healthcare is its ability to analyze vast amounts of data quickly and accurately. Traditional methods of data analysis in healthcare often involve manual processes that are time-consuming and prone to human error. AI, on the other hand, can sift through millions of patient records, medical images, and research papers in a fraction of the time it would take a human. This ability to process and analyze big data allows for more accurate diagnoses and more effective treatment plans. For example, consider the case of a patient presenting with symptoms that could indicate several different conditions. An AI system could analyze the patientΓÇÖs medical history, compare it with millions of other cases, and In recent years, artificial intelligence (AI) has revolutionized numerous industries, with healthcare being one of the most promising fields. The integration of AI in healthcare systems has the potential to transform patient care, diagnostics, and treatment plans. Imagine a world where AI-powered algorithms can predict diseases before they manifest, provide personalized treatment plans based on genetic information, and even assist in complex surgeries with unparalleled precision. One of the most significant advantages of AI in healthcare is its ability to analyze vast amounts of data quickly and accurately. Traditional methods of data analysis in healthcare often involve manual processes that are time-consuming and prone to human error. AI, on the other hand, can sift through millions of patient records, medical images, and research papers in a fraction of the time it would take a human. This ability to process and analyze big data allows for more accurate diagnoses and more effective treatment plans. For example, consider the case of a patient presenting with symptoms that could indicate several different conditions. An AI system could analyze the patientΓÇÖs medical history, compare it with millions of other cases, and suggest the most likely diagnosis. It could also recommend a personalized treatment plan based on the patientΓÇÖs genetic makeup, lifestyle, and other factors. This level of precision medicine has the potential to improve patient outcomes significantly. However, the implementation of AI in healthcare is not without its challenges. One of the main concerns is the ethical implications of using AI in such a sensitive field. For instance, who is responsible if an AI system makes a wrong diagnosis? How do we ensure that AI systems are not biased in their decision-making processes? Moreover, there are concerns about patient privacy and the security of sensitive medical data. Another challenge is the integration of AI systems into existing healthcare infrastructure. Many healthcare systems are already burdened with outdated technology and limited resources. Integrating advanced AI systems into these environments requires significant investment and training. Additionally, there is the question of how AI will impact the roles of healthcare professionals. While AI can assist doctors and nurses in their work, there is concern that it could also lead to job displacement. Despite these challenges, the potential benefits of AI in healthcare are immense. For example, AI-powered robots are already being used in some hospitals to assist with surgeries. These robots can perform delicate procedures with a level of precision that is difficult for humans to achieve. AI is also being used to develop new drugs and treatment plans. By analyzing the molecular structure of diseases, AI can help researchers identify potential treatments faster than traditional methods. Moreover, AI has In recent years, artificial intelligence (AI) has revolutionized numerous industries, with healthcare being one of the most promising fields. The integration of AI in healthcare systems has the potential to transform patient care, diagnostics, and treatment plans. Imagine a world where AI-powered algorithms can predict diseases before they manifest, provide personalized treatment plans based on genetic information, and even assist in complex surgeries with unparalleled precision. One of the most significant advantages of AI in healthcare is its ability to analyze vast amounts of data quickly and accurately. Traditional methods of data analysis in healthcare often involve manual processes that are time-consuming and prone to human error. AI, on the other hand, can sift through millions of patient records, medical images, and research papers in a fraction of the time it would take a human. This ability to process and analyze big data allows for more accurate diagnoses and more effective treatment plans. For example, consider the case of a patient presenting with symptoms that could indicate several different conditions. An AI system could analyze the patientΓÇÖs medical history, compare it with millions of other cases, and suggest the most likely diagnosis. It could also recommend a personalized treatment plan based on the patientΓÇÖs genetic makeup, lifestyle, and other factors. This level of precision medicine has the potential to improve patient outcomes significantly. However, the implementation of AI in healthcare is not without its challenges. One of the main concerns is the ethical implications of using AI in such a sensitive field. For instance, who is responsible if an AI system makes a wrong diagnosis? How do we ensure that AI systems are not biased in their decision-making processes? Moreover, there are concerns about patient privacy and the security of sensitive medical data. Another challenge is the integration of AI systems into existing healthcare infrastructure. Many healthcare systems are already burdened with outdated technology and limited resources. Integrating advanced AI systems into these environments requires significant investment and training. Additionally, there is the question of how AI will impact the roles of healthcare professionals. While AI can assist doctors and nurses in their work, there is concern that it could also lead to job displacement. Despite these challenges, the potential benefits of AI in healthcare are immense. For example, AI-powered robots are already being used in some hospitals to assist with surgeries. These robots can perform delicate procedures with a level of precision that is difficult for humans to achieve. AI is also being used to develop new drugs and treatment plans. By analyzing the molecular structure of diseases, AI can help researchers identify potential treatments faster than traditional methods. Moreover, AI has In recent years, artificial intelligence (AI) has revolutionized numerous industries, with healthcare being one of the most promising fields. The integration of AI in healthcare systems has the potential to transform patient care, diagnostics, and treatment plans. Imagine a world where AI-powered algorithms can predict diseases before they manifest, provide personalized treatment plans based on genetic information, and even assist in complex surgeries with unparalleled precision. One of the most significant advantages of AI in healthcare is its ability to analyze vast amounts of data quickly and accurately. Traditional methods of data analysis in healthcare often involve manual processes that are time-consuming and prone to human error. AI, on the other hand, can sift through millions of patient records, medical images, and research papers in a fraction of the time it would take a human. This ability to process and analyze big data allows for more accurate diagnoses and more effective treatment plans. For example, consider the case of a patient presenting with symptoms that could indicate several different conditions. An AI system could analyze the patientΓÇÖs medical history, compare it with millions of other cases, and suggest the most likely diagnosis. It could also recommend a personalized treatment plan based on the patientΓÇÖs genetic makeup, lifestyle, and other factors. This level of precision medicine has the potential to improve patient outcomes significantly. However, the implementation of AI in healthcare is not without its challenges. One of the main concerns is the ethical implications of using AI in such a sensitive field. For instance, who is responsible if an AI system makes a wrong diagnosis? How do we ensure that AI systems are not biased in their decision-making processes? Moreover, there are concerns about patient privacy and the security of sensitive medical data. Another challenge is the integration of AI systems into existing healthcare infrastructure. Many healthcare systems are already burdened with outdated technology and limited resources. Integrating advanced AI systems into these environments requires significant investment and training. Additionally, there is the question of how AI will impact the roles of healthcare professionals. While AI can assist doctors and nurses in their work, there is concern that it could also lead to job displacement. Despite these challenges, the potential benefits of AI in healthcare are immense. For example, AI-powered robots are already being used in some hospitals to assist with surgeries. These robots can perform delicate procedures with a level of precision that is difficult for humans to achieve. AI is also being used to develop new drugs and treatment plans. By analyzing the molecular structure of diseases, AI can help researchers identify potential treatments faster than traditional methods. Moreover, AI has In recent years, artificial intelligence (AI) has revolutionized numerous industries, with healthcare being one of the most promising fields. The integration of AI in healthcare systems has the potential to transform patient care, diagnostics, and treatment plans. Imagine a world where AI-powered algorithms can predict diseases before they manifest, provide personalized treatment plans based on genetic information, and even assist in complex surgeries with unparalleled precision. One of the most significant advantages of AI in healthcare is its ability to analyze vast amounts of data quickly and accurately. Traditional methods of data analysis in healthcare often involve manual processes that are time-consuming and prone to human error. AI, on the other hand, can sift through millions of patient records, medical images, and research papers in a fraction of the time it would take a human. This ability to process and analyze big data allows for more accurate diagnoses and more effective treatment plans. For example, consider the case of a patient presenting with symptoms that could indicate several different conditions. An AI system could analyze the patientΓÇÖs medical history, compare it with millions of other cases, and In recent years, artificial intelligence (AI) has revolutionized numerous industries, with healthcare being one of the most promising fields. The integration of AI in healthcare systems has the potential to transform patient care, diagnostics, and treatment plans. Imagine a world where AI-powered algorithms can predict diseases before they manifest, provide personalized treatment plans based on genetic information, and even assist in complex surgeries with unparalleled precision. One of the most significant advantages of AI in healthcare is its ability to analyze vast amounts of data quickly and accurately. Traditional methods of data analysis in healthcare often involve manual processes that are time-consuming and prone to human error. AI, on the other hand, can sift through millions of patient records, medical images, and research papers in a fraction of the time it would take a human. This ability to process and analyze big data allows for more accurate diagnoses and more effective treatment plans. For example, consider the case of a patient presenting with symptoms that could indicate several different conditions. An AI system could analyze the patientΓÇÖs medical history, compare it with millions of other cases, and suggest the most likely diagnosis. It could also recommend a personalized treatment plan based on the patientΓÇÖs genetic makeup, lifestyle, and other factors. This level of precision medicine has the potential to improve patient outcomes significantly. However, the implementation of AI in healthcare is not without its challenges. One of the main concerns is the ethical implications of using AI in such a sensitive field. For instance, who is responsible if an AI system makes a wrong diagnosis? How do we ensure that AI systems are not biased in their decision-making processes? Moreover, there are concerns about patient privacy and the security of sensitive medical data. Another challenge is the integration of AI systems into existing healthcare infrastructure. Many healthcare systems are already burdened with outdated technology and limited resources. Integrating advanced AI systems into these environments requires significant investment and training. Additionally, there is the question of how AI will impact the roles of healthcare professionals. While AI can assist doctors and nurses in their work, there is concern that it could also lead to job displacement. Despite these challenges, the potential benefits of AI in healthcare are immense. For example, AI-powered robots are already being used in some hospitals to assist with surgeries. These robots can perform delicate procedures with a level of precision that is difficult for humans to achieve. AI is also being used to develop new drugs and treatment plans. By analyzing the molecular structure of diseases, AI can help researchers identify potential treatments faster than traditional methods. Moreover, AI has In recent years, artificial intelligence (AI) has revolutionized numerous industries, with healthcare being one of the most promising fields. The integration of AI in healthcare systems has the potential to transform patient care, diagnostics, and treatment plans. Imagine a world where AI-powered algorithms can predict diseases before they manifest, provide personalized treatment plans based on genetic information, and even assist in complex surgeries with unparalleled precision. One of the most significant advantages of AI in healthcare is its ability to analyze vast amounts of data quickly and accurately. Traditional methods of data analysis in healthcare often involve manual processes that are time-consuming and prone to human error. AI, on the other hand, can sift through millions of patient records, medical images, and research papers in a fraction of the time it would take a human. This ability to process and analyze big data allows for more accurate diagnoses and more effective treatment plans. For example, consider the case of a patient presenting with symptoms that could indicate several different conditions. An AI system could analyze the patientΓÇÖs medical history, compare it with millions of other cases, and suggest the most likely diagnosis. It could also recommend a personalized treatment plan based on the patientΓÇÖs genetic makeup, lifestyle, and other factors. This level of precision medicine has the potential to improve patient outcomes significantly. However, the implementation of AI in healthcare is not without its challenges. One of the main concerns is the ethical implications of using AI in such a sensitive field. For instance, who is responsible if an AI system makes a wrong diagnosis? How do we ensure that AI systems are not biased in their decision-making processes? Moreover, there are concerns about patient privacy and the security of sensitive medical data. Another challenge is the integration of AI systems into existing healthcare infrastructure. Many healthcare systems are already burdened with outdated technology and limited resources. Integrating advanced AI systems into these environments requires significant investment and training. Additionally, there is the question of how AI will impact the roles of healthcare professionals. While AI can assist doctors and nurses in their work, there is concern that it could also lead to job displacement. Despite these challenges, the potential benefits of AI in healthcare are immense. For example, AI-powered robots are already being used in some hospitals to assist with surgeries. These robots can perform delicate procedures with a level of precision that is difficult for humans to achieve. AI is also being used to develop new drugs and treatment plans. By analyzing the molecular structure of diseases, AI can help researchers identify potential treatments faster than traditional methods. Moreover, AI has In recent years, artificial intelligence (AI) has revolutionized numerous industries, with healthcare being one of the most promising fields. The integration of AI in healthcare systems has the potential to transform patient care, diagnostics, and treatment plans. Imagine a world where AI-powered algorithms can predict diseases before they manifest, provide personalized treatment plans based on genetic information, and even assist in complex surgeries with unparalleled precision. One of the most significant advantages of AI in healthcare is its ability to analyze vast amounts of data quickly and accurately. Traditional methods of data analysis in healthcare often involve manual processes that are time-consuming and prone to human error. AI, on the other hand, can sift through millions of patient records, medical images, and research papers in a fraction of the time it would take a human. This ability to process and analyze big data allows for more accurate diagnoses and more effective treatment plans. For example, consider the case of a patient presenting with symptoms that could indicate several different conditions. An AI system could analyze the patientΓÇÖs medical history, compare it with millions of other cases, and suggest the most likely diagnosis. It could also recommend a personalized treatment plan based on the patientΓÇÖs genetic makeup, lifestyle, and other factors. This level of precision medicine has the potential to improve patient outcomes significantly. However, the implementation of AI in healthcare is not without its challenges. One of the main concerns is the ethical implications of using AI in such a sensitive field. For instance, who is responsible if an AI system makes a wrong diagnosis? How do we ensure that AI systems are not biased in their decision-making processes? Moreover, there are concerns about patient privacy and the security of sensitive medical data. Another challenge is the integration of AI systems into existing healthcare infrastructure. Many healthcare systems are already burdened with outdated technology and limited resources. Integrating advanced AI systems into these environments requires significant investment and training. Additionally, there is the question of how AI will impact the roles of healthcare professionals. While AI can assist doctors and nurses in their work, there is concern that it could also lead to job displacement. Despite these challenges, the potential benefits of AI in healthcare are immense. For example, AI-powered robots are already being used in some hospitals to assist with surgeries. These robots can perform delicate procedures with a level of precision that is difficult for humans to achieve. AI is also being used to develop new drugs and treatment plans. By analyzing the molecular structure of diseases, AI can help researchers identify potential treatments faster than traditional methods. Moreover, AI has In recent years, artificial intelligence (AI) has revolutionized numerous industries, with healthcare being one of the most promising fields. The integration of AI in healthcare systems has the potential to transform patient care, diagnostics, and treatment plans. Imagine a world where AI-powered algorithms can predict diseases before they manifest, provide personalized treatment plans based on genetic information, and even assist in complex surgeries with unparalleled precision. One of the most significant advantages of AI in healthcare is its ability to analyze vast amounts of data quickly and accurately. Traditional methods of data analysis in healthcare often involve manual processes that are time-consuming and prone to human error. AI, on the other hand, can sift through millions of patient records, medical images, and research papers in a fraction of the time it would take a human. This ability to process and analyze big data allows for more accurate diagnoses and more effective treatment plans. For example, consider the case of a patient presenting with symptoms that could indicate several different conditions. An AI system could analyze the patientΓÇÖs medical history, compare it with millions of other cases, and 

special_tokens_map.json

@@ -0,0 +1,6 @@
+{
+  "bos_token": "<s>",
+  "eos_token": "</s>",
+  "pad_token": "<unk>",
+  "unk_token": "<unk>"
+}

tokenization_chatglm.py

@@ -0,0 +1,506 @@
+import json
+import os
+import re
+from typing import List, Optional, Union, Dict
+from sentencepiece import SentencePieceProcessor
+from transformers import AddedToken, PreTrainedTokenizer, PreTrainedTokenizerFast
+from transformers.convert_slow_tokenizer import (
+    SLOW_TO_FAST_CONVERTERS,
+    SpmConverter,
+    decoders,
+    normalizers,
+    pre_tokenizers,
+    processors,
+)
+from transformers.utils import logging, PaddingStrategy
+from transformers.tokenization_utils_base import EncodedInput, BatchEncoding
+
+
+logger = logging.get_logger(__name__)
+
+ADDITIONAL_SPECIAL_TOKENS = [
+    "[MASK]",
+    "[gMASK]",
+    "[sMASK]",
+    "<!sop!>",
+    "<!eop!>",
+    "<|system|>",
+    "<|user|>",
+    "<|assistant|>",
+    "<|observation|>",
+]
+PREFIX_TOKENS = ["[gMASK]", "<!sop!>"]
+
+DUMMY_PREFIX_INDICATOR_FOR_FAST = "<!dummy-prefix!>"
+
+
+class SPTokenizer:
+    def __init__(self, model_path: str):
+        # reload tokenizer
+        assert os.path.isfile(model_path), model_path
+        self.sp_model = SentencePieceProcessor(model_file=model_path)
+
+        # BOS / EOS token IDs
+        self.n_words: int = self.sp_model.vocab_size()
+        self.bos_id: int = self.sp_model.bos_id()
+        self.eos_id: int = self.sp_model.eos_id()
+        self.pad_id: int = self.sp_model.unk_id()
+        assert self.sp_model.vocab_size() == self.sp_model.get_piece_size()
+
+        special_tokens = ADDITIONAL_SPECIAL_TOKENS
+        self.special_tokens = {}
+        self.index_special_tokens = {}
+        for token in special_tokens:
+            self.special_tokens[token] = self.n_words
+            self.index_special_tokens[self.n_words] = token
+            self.n_words += 1
+        self.role_special_token_expression = "|".join([re.escape(token) for token in special_tokens]) # for apply_chat_template
+
+    def tokenize(self, s: str, encode_special_tokens=False):
+        if encode_special_tokens:
+            last_index = 0
+            t = []
+            for match in re.finditer(self.role_special_token_expression, s):
+                if last_index < match.start():
+                    t.extend(self.sp_model.EncodeAsPieces(s[last_index:match.start()]))
+                t.append(s[match.start():match.end()])
+                last_index = match.end()
+            if last_index < len(s):
+                t.extend(self.sp_model.EncodeAsPieces(s[last_index:]))
+            return t
+        else:
+            return self.sp_model.EncodeAsPieces(s)
+
+    def encode(self, s: str, bos: bool = False, eos: bool = False) -> List[int]:
+        assert type(s) is str
+        t = self.sp_model.encode(s)
+        if bos:
+            t = [self.bos_id] + t
+        if eos:
+            t = t + [self.eos_id]
+        return t
+
+    def decode(self, t: List[int]) -> str:
+        text, buffer = "", []
+        for token in t:
+            if token in self.index_special_tokens:
+                if buffer:
+                    text += self.sp_model.decode(buffer)
+                    buffer = []
+                text += self.index_special_tokens[token]
+            else:
+                buffer.append(token)
+        if buffer:
+            text += self.sp_model.decode(buffer)
+        return text
+
+    def decode_tokens(self, tokens: List[str]) -> str:
+        text = self.sp_model.DecodePieces(tokens)
+        return text
+
+    def convert_token_to_id(self, token):
+        """ Converts a token (str) in an id using the vocab. """
+        if token in self.special_tokens:
+            return self.special_tokens[token]
+        return self.sp_model.PieceToId(token)
+
+    def convert_id_to_token(self, index):
+        """Converts an index (integer) in a token (str) using the vocab."""
+        if index in self.index_special_tokens:
+            return self.index_special_tokens[index]
+        if index in [self.eos_id, self.bos_id, self.pad_id] or index < 0 or index >= self.sp_model.vocab_size():
+            return ""
+        return self.sp_model.IdToPiece(index)
+
+
+class ChatGLMTokenizer(PreTrainedTokenizer):
+
+    vocab_files_names = {"vocab_file": "tokenizer.model"}
+    model_input_names = ["input_ids", "attention_mask", "position_ids"]
+
+    def __init__(
+        self,
+        vocab_file,
+        padding_side="left",
+        clean_up_tokenization_spaces=False,
+        encode_special_tokens=False,
+        **kwargs
+    ):
+        self.name = "GLMTokenizer"
+        self.vocab_file = vocab_file
+        self.tokenizer = SPTokenizer(vocab_file)
+        self.special_tokens = {
+            "<bos>": self.tokenizer.bos_id,
+            "<eos>": self.tokenizer.eos_id,
+            "<unk>": self.tokenizer.pad_id,
+            "<pad>": self.tokenizer.pad_id
+        }
+        self.encode_special_tokens = encode_special_tokens
+
+        super().__init__(
+            padding_side=padding_side,
+            clean_up_tokenization_spaces=clean_up_tokenization_spaces,
+            **kwargs
+        )
+
+    def get_command(self, token):
+        if token in self.special_tokens:
+            return self.special_tokens[token]
+        assert token in self.tokenizer.special_tokens, f"{token} is not a special token for {self.name}"
+        return self.tokenizer.special_tokens[token]
+
+    @property
+    def unk_token(self) -> str:
+        return self.tokenizer.sp_model.IdToPiece(self.get_command("<unk>"))
+
+    @property
+    def pad_token(self) -> str:
+        return self.tokenizer.sp_model.IdToPiece(self.get_command("<pad>"))
+
+    @property
+    def eos_token(self) -> str:
+        return self.tokenizer.sp_model.IdToPiece(self.get_command("<eos>"))
+
+    @property
+    def unk_token_id(self) -> int:
+        return self.get_command("<unk>")
+
+    @property
+    def pad_token_id(self) -> int:
+        return self.get_command("<pad>")
+
+    @property
+    def eos_token_id(self):
+        return self.get_command("<eos>")
+
+    @unk_token.setter
+    def unk_token(self, value):
+        logger.warning("Setting unk_token is not supported, use the default one.")
+
+    @pad_token.setter
+    def pad_token(self, value):
+        logger.warning("Setting pad_token is not supported, use the default one.")
+
+    @eos_token.setter
+    def eos_token(self, value):
+        logger.warning("Setting eos_token is not supported, use the default one.")
+
+    @property
+    def vocab_size(self):
+        return self.tokenizer.n_words
+
+    def get_vocab(self):
+        """ Returns vocab as a dict """
+        vocab = {self._convert_id_to_token(i): i for i in range(self.vocab_size)}
+        vocab.update(self.added_tokens_encoder)
+        return vocab
+
+    def _tokenize(self, text, **kwargs):
+        return self.tokenizer.tokenize(text, encode_special_tokens=self.encode_special_tokens)
+
+    def _convert_token_to_id(self, token):
+        """ Converts a token (str) in an id using the vocab. """
+        return self.tokenizer.convert_token_to_id(token)
+
+    def _convert_id_to_token(self, index):
+        """Converts an index (integer) in a token (str) using the vocab."""
+        return self.tokenizer.convert_id_to_token(index)
+
+    def convert_tokens_to_string(self, tokens: List[str]) -> str:
+        return self.tokenizer.decode_tokens(tokens)
+
+    def save_vocabulary(self, save_directory, filename_prefix=None):
+        """
+        Save the vocabulary and special tokens file to a directory.
+
+        Args:
+            save_directory (`str`):
+                The directory in which to save the vocabulary.
+            filename_prefix (`str`, *optional*):
+                An optional prefix to add to the named of the saved files.
+
+        Returns:
+            `Tuple(str)`: Paths to the files saved.
+        """
+        if os.path.isdir(save_directory):
+            vocab_file = os.path.join(
+                save_directory, self.vocab_files_names["vocab_file"]
+            )
+        else:
+            vocab_file = save_directory
+
+        with open(self.vocab_file, 'rb') as fin:
+            proto_str = fin.read()
+
+        with open(vocab_file, "wb") as writer:
+            writer.write(proto_str)
+
+        return (vocab_file,)
+
+    def get_prefix_tokens(self):
+        return list(map(self.get_command, PREFIX_TOKENS))
+
+    def build_single_message(self, role, metadata, message):
+        assert role in ["system", "user", "assistant", "observation"], role
+        role_tokens = [self.get_command(f"<|{role}|>")] + self.tokenizer.encode(f"{metadata}\n")
+        message_tokens = self.tokenizer.encode(message)
+        tokens = role_tokens + message_tokens
+        return tokens
+
+    def build_chat_input(self, query, history=None, role="user"):
+        if history is None:
+            history = []
+        input_ids = []
+        for item in history:
+            content = item["content"]
+            if item["role"] == "system" and "tools" in item:
+                content = content + "\n" + json.dumps(item["tools"], indent=4, ensure_ascii=False)
+            input_ids.extend(self.build_single_message(item["role"], item.get("metadata", ""), content))
+        input_ids.extend(self.build_single_message(role, "", query))
+        input_ids.extend([self.get_command("<|assistant|>")])
+        return self.batch_encode_plus([input_ids], return_tensors="pt", is_split_into_words=True)
+
+    def build_inputs_with_special_tokens(
+        self, token_ids_0: List[int], token_ids_1: Optional[List[int]] = None
+    ) -> List[int]:
+        """
+        Build model inputs from a sequence or a pair of sequence for sequence classification tasks by concatenating and
+        adding special tokens. A BERT sequence has the following format:
+
+        - single sequence: `[CLS] X [SEP]`
+        - pair of sequences: `[CLS] A [SEP] B [SEP]`
+
+        Args:
+            token_ids_0 (`List[int]`):
+                List of IDs to which the special tokens will be added.
+            token_ids_1 (`List[int]`, *optional*):
+                Optional second list of IDs for sequence pairs.
+
+        Returns:
+            `List[int]`: List of [input IDs](../glossary#input-ids) with the appropriate special tokens.
+        """
+        prefix_tokens = self.get_prefix_tokens()
+        token_ids_0 = prefix_tokens + token_ids_0
+        if token_ids_1 is not None:
+            token_ids_0 = token_ids_0 + token_ids_1 + [self.get_command("<eos>")]
+        return token_ids_0
+
+    def _pad(
+        self,
+        encoded_inputs: Union[Dict[str, EncodedInput], BatchEncoding],
+        max_length: Optional[int] = None,
+        padding_strategy: PaddingStrategy = PaddingStrategy.DO_NOT_PAD,
+        pad_to_multiple_of: Optional[int] = None,
+        return_attention_mask: Optional[bool] = None,
+    ) -> dict:
+        """
+        Pad encoded inputs (on left/right and up to predefined length or max length in the batch)
+
+        Args:
+            encoded_inputs:
+                Dictionary of tokenized inputs (`List[int]`) or batch of tokenized inputs (`List[List[int]]`).
+            max_length: maximum length of the returned list and optionally padding length (see below).
+                Will truncate by taking into account the special tokens.
+            padding_strategy: PaddingStrategy to use for padding.
+
+                - PaddingStrategy.LONGEST Pad to the longest sequence in the batch
+                - PaddingStrategy.MAX_LENGTH: Pad to the max length (default)
+                - PaddingStrategy.DO_NOT_PAD: Do not pad
+                The tokenizer padding sides are defined in self.padding_side:
+
+                    - 'left': pads on the left of the sequences
+                    - 'right': pads on the right of the sequences
+            pad_to_multiple_of: (optional) Integer if set will pad the sequence to a multiple of the provided value.
+                This is especially useful to enable the use of Tensor Core on NVIDIA hardware with compute capability
+                `>= 7.5` (Volta).
+            return_attention_mask:
+                (optional) Set to False to avoid returning attention mask (default: set to model specifics)
+        """
+        # Load from model defaults
+        assert self.padding_side == "left"
+
+        required_input = encoded_inputs[self.model_input_names[0]]
+        seq_length = len(required_input)
+
+        if padding_strategy == PaddingStrategy.LONGEST:
+            max_length = len(required_input)
+
+        if max_length is not None and pad_to_multiple_of is not None and (max_length % pad_to_multiple_of != 0):
+            max_length = ((max_length // pad_to_multiple_of) + 1) * pad_to_multiple_of
+
+        needs_to_be_padded = padding_strategy != PaddingStrategy.DO_NOT_PAD and len(required_input) != max_length
+
+        # Initialize attention mask if not present.
+        if "attention_mask" not in encoded_inputs:
+            encoded_inputs["attention_mask"] = [1] * seq_length
+
+        if "position_ids" not in encoded_inputs:
+            encoded_inputs["position_ids"] = list(range(seq_length))
+
+        if needs_to_be_padded:
+            difference = max_length - len(required_input)
+
+            if "attention_mask" in encoded_inputs:
+                encoded_inputs["attention_mask"] = [0] * difference + encoded_inputs["attention_mask"]
+            if "position_ids" in encoded_inputs:
+                encoded_inputs["position_ids"] = [0] * difference + encoded_inputs["position_ids"]
+            encoded_inputs[self.model_input_names[0]] = [self.pad_token_id] * difference + required_input
+
+        return encoded_inputs
+
+
+class ChatGLMTokenizerFast(PreTrainedTokenizerFast):
+    # multiple breaking changes, no backward-compatibility
+    slow_tokenizer_class = ChatGLMTokenizer
+    vocab_files_names = {
+        **ChatGLMTokenizer.vocab_files_names,
+        **PreTrainedTokenizerFast.vocab_files_names,
+    }
+
+    def __init__(self, **kwargs):
+        kwargs.setdefault("clean_up_tokenization_spaces", False)
+        kwargs.setdefault("bos_token", "<s>")
+        kwargs.setdefault("eos_token", "</s>")
+        kwargs.setdefault("unk_token", "<unk>")
+        kwargs.setdefault("pad_token", "<unk>")
+        super().__init__(**kwargs)
+
+    @property
+    def dummy_prefix_indicator(self):
+        return DUMMY_PREFIX_INDICATOR_FOR_FAST
+
+    @property
+    def can_save_slow_tokenizer(self) -> bool:
+        # multiple breaking changes
+        return False
+
+    def save_pretrained(self, *args, **kwargs):
+        if not self.can_save_slow_tokenizer:
+            logger.warning(
+                f"{type(self).__name__} does not support saving slow tokenizer. "
+                "Saving it at the same directory may break the original tokenizer. "
+                "Please keep a backup beforehand."
+            )
+
+        return super().save_pretrained(*args, **kwargs)
+
+    def build_single_message_prompt(self, role, metadata, message):
+        assert role in ["system", "user", "assistant", "observation"], role
+        return (
+            f"<|{role}|>"
+            f"{self.dummy_prefix_indicator}{metadata}\n"
+            f"{self.dummy_prefix_indicator}{message}"
+        )
+
+    def build_chat_prompt(self, query, history=None, role="user", metadata=""):
+        inputs = []
+
+        for item in history or []:
+            content = item["content"]
+
+            if item["role"] == "system" and "tools" in item:
+                content += "\n" + json.dumps(
+                    item["tools"], indent=4, ensure_ascii=False
+                )
+
+            inputs.append(
+                self.build_single_message_prompt(
+                    item["role"], item.get("metadata", ""), content
+                )
+            )
+
+        inputs.append(self.build_single_message_prompt(role, metadata, query))
+        inputs.append("<|assistant|>")
+
+        return "".join(inputs)
+
+    def build_chat_input(self, *args, **kwargs):
+        return self.batch_encode_plus(
+            [self.build_chat_prompt(*args, **kwargs)],
+            return_tensors="pt",
+        )
+
+
+ChatGLMTokenizer.register_for_auto_class()
+ChatGLMTokenizerFast.register_for_auto_class()
+
+
+class ChatGLMTokenizerConverter(SpmConverter):
+    handle_byte_fallback = True
+
+    def normalizer(self, proto):
+        return normalizers.Sequence(
+            [
+                normalizers.Replace(
+                    pattern=DUMMY_PREFIX_INDICATOR_FOR_FAST, content="▁"
+                ),
+                normalizers.Replace(pattern=" ", content="▁"),
+            ]
+        )
+
+    def pre_tokenizer(self, replacement, add_prefix_space):
+        # NOTE: don't use Metaspace, it won't merge spaces into one token
+        # without Metaspace: "  " => ["▁▁"]
+        # with Metaspace: "  " => ["▁", "▁"]
+        return pre_tokenizers.Split(DUMMY_PREFIX_INDICATOR_FOR_FAST, "merged_with_next")
+
+    def decoder(self, replacement, add_prefix_space):
+        return decoders.Sequence(
+            [
+                decoders.ByteFallback(),
+                decoders.Metaspace(replacement="▁", add_prefix_space=True),
+            ]
+        )
+
+    def tokenizer(self, proto):
+        tokenizer = super().tokenizer(proto)
+
+        tokenizer.model.byte_fallback = True
+
+        assert tokenizer.token_to_id("<unk>") == 0
+        assert tokenizer.token_to_id("<s>") == 1
+        assert tokenizer.token_to_id("</s>") == 2
+        special_tokens = [
+            "<unk>",
+            "<s>",
+            "</s>",
+            *ADDITIONAL_SPECIAL_TOKENS,
+        ]
+
+        tokenizer.add_special_tokens(
+            [AddedToken(token, special=True) for token in special_tokens]
+        )
+
+        return tokenizer
+
+    def converted(self):
+        tokenizer = super().converted()
+
+        # Post processors
+        prefix_token_ids = list(map(tokenizer.token_to_id, PREFIX_TOKENS))
+        assert all(i is not None for i in prefix_token_ids)
+        prefix_template = " ".join(PREFIX_TOKENS)
+
+        template_special_tokens = list(frozenset(zip(PREFIX_TOKENS, prefix_token_ids)))
+
+        if "</s>" not in PREFIX_TOKENS:
+            eos_token_id = tokenizer.token_to_id("</s>")
+            assert eos_token_id is not None
+            template_special_tokens.append(("</s>", eos_token_id))
+
+        post = processors.TemplateProcessing(
+            single=f"{prefix_template} $A",
+            pair=f"{prefix_template} $A $B:1 </s>:1",
+            special_tokens=template_special_tokens,
+        )
+        if tokenizer.post_processor is None:
+            tokenizer.post_processor = post
+        else:
+            tokenizer.post_processor = processors.Sequence(
+                [tokenizer.post_processor, post]
+            )
+
+        return tokenizer
+
+
+SLOW_TO_FAST_CONVERTERS[ChatGLMTokenizer.__name__] = ChatGLMTokenizerConverter

tokenizer.model

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e7dc4c393423b76e4373e5157ddc34803a0189ba96b21ddbb40269d31468a6f2
+size 1018370

tokenizer_config.json

@@ -0,0 +1,117 @@
+{
+  "added_tokens_decoder": {
+    "0": {
+      "content": "<unk>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "1": {
+      "content": "<s>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "2": {
+      "content": "</s>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "64789": {
+      "content": "[MASK]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "64790": {
+      "content": "[gMASK]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "64791": {
+      "content": "[sMASK]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "64792": {
+      "content": "<!sop!>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "64793": {
+      "content": "<!eop!>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "64794": {
+      "content": "<|system|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "64795": {
+      "content": "<|user|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "64796": {
+      "content": "<|assistant|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "64797": {
+      "content": "<|observation|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    }
+  },
+  "auto_map": {
+    "AutoTokenizer": [
+      "tokenization_chatglm.ChatGLMTokenizer",
+      "tokenization_chatglm.ChatGLMTokenizerFast"
+    ]
+  },
+  "bos_token": "<s>",
+  "chat_template": "{% for message in messages %}{% if loop.first %}[gMASK]sop<|{{ message['role'] }}|> \n {{ message['content'] }}{% else %}<|{{ message['role'] }}|> \n {{ message['content'] }}{% endif %}{% endfor %}{% if add_generation_prompt %}<|assistant|>{% endif %}",
+  "clean_up_tokenization_spaces": false,
+  "do_lower_case": false,
+  "eos_token": "</s>",
+  "model_max_length": 1000000000000000019884624838656,
+  "pad_token": "<unk>",
+  "padding_side": "left",
+  "remove_space": false,
+  "tokenizer_class": "ChatGLMTokenizer",
+  "unk_token": "<unk>"
+}