Delete op_tokenizer.py

op_tokenizer.py

@@ -1,268 +0,0 @@
-import json
-from typing import List, Optional, Dict
-from transformers import PreTrainedTokenizer
-import os
-import json
-import re
-default_config = {
-    "custom_digits": "0123456789ABCDEF",
-    "variable_atoms": {
-        "left_operand": "a",   # 左操作数变量名
-        "right_operand": "b"   # 右操作数变量名
-    },
-
-    "other_symbols_atoms": {
-        "left_parenthesis": "(",    # 左括号
-        "right_parenthesis": ")",   # 右括号
-        "equals_sign": "=",         # 等号，常用于赋值或比较
-        "nan_symbol": "NaN",        # 非数（Not a Number）
-        "inf_symbol": "Inf"         # 无穷大（Infinity）
-    },
-
-    "operator_symbol_min_len": 1,
-    "operator_symbol_max_len": 3,
-
-    "basic_operator_symbols": ["+", "-", "*", "/", "%"],
-
-    "base_symbols": [
-        "≮⫘↔", 
-        "⫏≰", 
-        "⪩⨒∯", 
-        "⇑⪆", 
-        "↹⩛", 
-        "≴∭⊉", 
-        "⪪⊹⋣", 
-        "⋋%⋟", 
-        "⊺⇮", 
-        "⋰*⋻", 
-        "⫖↰⪸", 
-        "⪎⋱⫍", 
-        "⨗⨭⨅", 
-        "⫶⩼⫲", 
-        "∃⊬"
-    ],
-
-    "comparison_ops": ["==", ">", "<", ">=", "<=", "!="],
-
-    "logical_connectors": ["and", "or"],
-
-    "definition_symbols": [
-        ",",    
-        ";",    
-        "if",   
-        "else", 
-        "{",    
-        "}",    
-        "abs"   
-    ]
-}
-
-class OpTokenizer(PreTrainedTokenizer):
-    def __init__(self, vocab_file, **kwargs):
-       
-        self.param_config= default_config
-        self.vocab = self.load_vocab(vocab_file)
-        self.ids_to_tokens = {v: k for k, v in self.vocab.items()}
-        super().__init__(**kwargs)
-        # 定义基础符号
-        self.basic_symbols = list("0123456789()=ABCDEFab")
-        self.special_results = ['NaN', 'Inf']
-        self.comparison_ops = ["==", ">", "<", ">=", "<=", "!="]
-        self.logical_connectors = ["and", "or"]
-        self.definition_symbols = [",", ";", "if", "else", "{", "}", "abs"]
-
-        self.token_regex = self.build_token_regex()
-        
-        # 初始化特殊标记 ID
-        self.pad_id = self.vocab['[PAD]']
-        self.unk_id = self.vocab['[UNK]']
-        self.sep_id = self.vocab['[SEP]']
-        self.mask_id = self.vocab['[MASK]']
-        self.bos_id = self.vocab['[BOS]']
-        self.eos_id = self.vocab['[EOS]']
-        self.eod_id = self.vocab['[EOD]']
-        
-    def load_vocab(self, vocab_file):
-        # 实现你的词表加载逻辑
-        with open(vocab_file, encoding="utf-8") as f:
-            vocab = json.load(f)
-        return vocab
-
-    def save_vocabulary(self, save_directory, filename_prefix=""):
-        if filename_prefix is None:
-            filename_prefix = ""
-
-        if not os.path.exists(save_directory):
-            os.makedirs(save_directory)
-
-        vocab_file_path = os.path.join(save_directory, filename_prefix + "vocab.json")
-
-        with open(vocab_file_path, "w", encoding="utf-8") as f:
-            json.dump(self.vocab, f, ensure_ascii=False, indent=4)
-
-        print(f"Vocabulary saved to {vocab_file_path}")
-        
-        return (vocab_file_path,)  # 返回元组而不是列表
-
-    def build_token_regex(self):
-        """构建分词正则表达式，逐字符、符号进行匹配"""
-        # 特殊结果的正则表达式（比如 NaN, Inf）
-        special_results = [re.escape(result) for result in self.special_results]
-        # 比较操作符的正则表达式
-        comparison_ops = [re.escape(op) for op in self.comparison_ops]
-        # 逻辑连接符的正则表达式
-        logical_connectors = [re.escape(connector) for connector in self.logical_connectors]
-        
-        operator_pattern = r"(?P<OPERATOR>([+\-*/%]|[\u2200-\u22FF\u2A00-\u2BFF\u2190-\u21FF])+)"
-        variable_pattern = r"(?P<VARIABLE>[a-b])"
-        digit_pattern = r"(?P<DIGIT>[0-9A-F])"
-        special_result_pattern = r"(?P<SPECIAL_RESULT>" + "|".join(special_results) + ")"
-        comparison_ops_pattern = r"(?P<COMPARISON_OP>" + "|".join(comparison_ops) + ")"
-        logical_connectors_pattern = r"(?P<LOGICAL_CONNECTOR>" + "|".join(logical_connectors) + ")"
-        if_else_pattern = r"(?P<IF_ELSE>if|else)"
-        whitespace_pattern = r"(?P<WHITESPACE>\s+)"
-        abs_pattern =  r"(?P<ABS>abs)"
-        punctuation_patterns = [
-                r"(?P<PARENTHESIS_LEFT>\()",
-                r"(?P<PARENTHESIS_RIGHT>\))",
-                r"(?P<CURLY_BRACE_LEFT>{)",
-                r"(?P<CURLY_BRACE_RIGHT>})",
-                r"(?P<SEMICOLON>;)",
-                r"(?P<COMMA>,)",
-                r"(?P<EQUAL>=)"
-        ]
-        
-        # 所有模式结合在一起，注意先后顺序，应该先匹配长的
-        token_patterns = [
-            operator_pattern,
-            special_result_pattern,  # 特殊符号（如 NaN, Inf）
-            comparison_ops_pattern,  # 比较操作符
-            logical_connectors_pattern,  # 逻辑连接符
-            if_else_pattern,  # if 和 else
-            abs_pattern,
-            digit_pattern, 
-            variable_pattern,  # 小写字母（变量名）
-            whitespace_pattern,  # 空格和换行符
-
-        ] + punctuation_patterns  # 将标点符号的正则表达式添加到列表中
-        
-        # 使用 | 连接所有模式
-        combined_pattern = "|".join(token_patterns)
-        
-        # 返回编译后的正则表达式对象
-        return re.compile(combined_pattern)
-    
-    def tokenize(self, text: str, mode: str = 'text', add_special_tokens: bool = True):
-        if mode == 'definition':
-            return self._tokenize_definition(text, add_special_tokens)
-        elif mode == 'text':
-            return self._tokenize_equation(text, add_special_tokens)
-        elif mode == 'withdef_text':
-            return self._tokenize_withdef_text(text, add_special_tokens)
-        else:
-            raise ValueError(f"Unsupported mode: {self.mode}")
-
-    def _tokenize_definition(self, text, add_special_tokens):
-        tokens = []
-        if add_special_tokens:
-            tokens.append('[DEF_START]')
-        for match in self.token_regex.finditer(text):
-            token_type = match.lastgroup
-            token_value = match.group(token_type)
-            if token_type != "WHITESPACE":
-                tokens.append(token_value)
-        if add_special_tokens:
-            tokens.append('[DEF_END]')
-        return tokens
-
-    def _tokenize_equation(self, text, add_special_tokens):
-        tokens = []
-        if add_special_tokens:
-            tokens.append('[EQ_START]')
-        
-        self.digit_pattern = f"[{re.escape(self.param_config['custom_digits'])}]"
-        self.number_pattern = f"[-]?{self.digit_pattern}+"
-        self.base_symbols_pattern = f"(?:{'|'.join(map(re.escape, self.param_config['base_symbols']))})"
-        self.base_symbols_number_pattern = f"({self.base_symbols_pattern}{self.number_pattern})"
-
-        parts = re.split(self.base_symbols_number_pattern, text)
-        final_parts = []
-        for part in parts:
-            if re.search(self.number_pattern, part):
-                sub_parts = re.split(f"({self.number_pattern})", part)
-                final_parts.extend(sub_parts)
-            else:
-                final_parts.append(part)
-
-        for part in final_parts:
-            for match in self.token_regex.finditer(part):
-                token_type = match.lastgroup
-                token_value = match.group(token_type)
-                if token_type != "WHITESPACE":
-                    tokens.append(token_value)
-
-        if add_special_tokens:
-            tokens.append('[EQ_END]')
-        return tokens
-
-    def _tokenize_withdef_text(self, text, add_special_tokens):
-        tokens = []
-        segments = re.split(r'(\[DEF_START\]|\[DEF_JOIN\]|\[DEF_END\]|\[EQ_START\]|\[EQ_END\])', text)
-        current_mode = None
-
-        for seg in segments:
-            seg = seg.strip()
-            if not seg:
-                continue
-
-            if seg in ['[DEF_START]', '[DEF_JOIN]']:
-                if add_special_tokens:
-                    tokens.append(seg)
-                current_mode = 'definition'
-            elif seg == '[DEF_END]':
-                if add_special_tokens:
-                    tokens.append(seg)
-                current_mode = None
-            elif seg == '[EQ_START]':
-                if add_special_tokens:
-                    tokens.append(seg)
-                current_mode = 'text'
-            elif seg == '[EQ_END]':
-                if add_special_tokens:
-                    tokens.append(seg)
-                current_mode = None
-            else:
-                if current_mode == 'definition':
-                    inner_tokens = self._tokenize_definition(seg, add_special_tokens=False)
-                    tokens.extend(inner_tokens)
-                elif current_mode == 'text':
-                    inner_tokens = self._tokenize_equation(seg, add_special_tokens=False)
-                    tokens.extend(inner_tokens)
-                else:
-                    tokens.extend(seg.split())
-        return tokens
-
-
-    def convert_tokens_to_ids(self, tokens):
-        if isinstance(tokens[0], str):
-            return [self.vocab.get(token, self.vocab['[UNK]']) for token in tokens]
-        return tokens
-
-    def convert_ids_to_tokens(self, ids):
-        reverse_vocab = {v: k for k, v in self.vocab.items()}
-        return [reverse_vocab.get(i, '[UNK]') for i in ids]
-    
-    def encode(self, text, mode=None, add_special_tokens=None):
-        tokens = self.tokenize(text, mode=mode, add_special_tokens=add_special_tokens)
-        return self.convert_tokens_to_ids(tokens)
-
-
-    def decode(self, ids, skip_special_tokens=False):
-        tokens = self.convert_ids_to_tokens(ids)
-        if skip_special_tokens:
-            tokens = [t for t in tokens if not (t.startswith('[') and t.endswith(']'))]
-        return " ".join(tokens).replace(" ##", "")
-
-    def get_vocab(self):
-        return self.vocab
-