Remove tokenizer.json and replace tokenizer.py with correct version.

tokenizer.json

@@ -1,129 +0,0 @@
-# based on https://github.com/EleutherAI/gpt-neox/blob/main/megatron/tokenizer/tokenizer.py
-from __future__ import annotations
-
-import torch
-
-import numpy as np
-
-from os import PathLike
-from typing import List, Tuple
-
-from tokenizers import Tokenizer
-from transformers.tokenization_utils import PreTrainedTokenizer
-from transformers.tokenization_utils_base import BatchEncoding, TruncationStrategy
-from transformers.utils.generic import TensorType, PaddingStrategy
-
-
-EMPTY: str = ""
-
-
-class ByteTokenizer(PreTrainedTokenizer):
-
-    """UTF-8 Encoder."""
-
-    @classmethod
-    def from_pretrained(cls, model_id: str | PathLike, **kwargs) -> ByteTokenizer:
-
-        return cls(**kwargs, byte_level=True)
-
-    @property
-    def vocab_size(self) -> int:
-
-        return 512
-
-    @property
-    def byte_level(self) -> bool:
-
-        return self.init_kwargs.get('byte_level', True)
-
-    def get_vocab(self) -> Dict[str, int]:
-
-        return {chr(i): i for i in range(self.vocab_size)}
-
-    def __len__(self) -> int:
-
-        return self.vocab_size
-
-    def clamp(self, n: int) -> int:
-
-        return max(32, min(n, self.vocab_size))
-
-    def _tokenize(self, text: str, **kwargs) -> List[str]:
-
-        return list(text)
-
-    def byte_tokenize(self, text: str) -> np.ndarray:
-
-        return np.frombuffer(text.encode('utf-8'), dtype=np.uint8)
-
-    def _convert_token_to_id(self, token: str) -> int:
-
-        return self.clamp(ord(token))
-
-    def _convert_id_to_token(self, index: int) -> str:
-
-        return chr(self.clamp(index))
-
-    def convert_tokens_to_string(self, tokens: List[str]) -> str:
-
-        return EMPTY.join(tokens)
-
-    def _decode(self, token_ids: List[int], **kwargs) -> str:
-
-        indices = np.asarray(token_ids, dtype=np.uint8)
-
-        return (
-            indices.clip(min=32, max=self.vocab_size, out=indices)
-            .tobytes()
-            .decode('utf-8')
-        )
-
-    def _encode_plus(self, text: str, **kwargs) -> BatchEncoding:
-
-        first_ids = self.byte_tokenize(text).tolist()
-
-        return self.prepare_for_model(
-            first_ids,
-            pair_ids=None,
-            add_special_tokens=kwargs.get('add_special_tokens', False),
-            padding=kwargs.get('padding_strategy', PaddingStrategy.DO_NOT_PAD).value,
-            truncation=kwargs.get('truncation_strategy', TruncationStrategy.DO_NOT_TRUNCATE).value,
-            max_length=kwargs.get('max_length'),
-            stride=kwargs.get('stride', 0),
-            pad_to_multiple_of=kwargs.get('pad_to_multiple_of'),
-            return_tensors=kwargs.get('return_tensors'),
-            prepend_batch_axis=True,
-            return_attention_mask=kwargs.get('return_attention_mask'),
-            return_token_type_ids=kwargs.get('return_token_type_ids'),
-            return_overflowing_tokens=kwargs.get('return_overflowing_tokens', False),
-            return_special_tokens_mask=kwargs.get('return_special_tokens_mask', False),
-            return_length=kwargs.get('return_length', False),
-            verbose=kwargs.get('verbose', True),
-        )
-
-    def _batch_encode_plus(self, batch_text: List[str], **kwargs) -> BatchEncoding:
-
-        input_ids = [(self.byte_tokenize(text).tolist(), None) for text in batch_text]
-
-        return self._batch_prepare_for_model(
-            input_ids,
-            add_special_tokens=kwargs.get('add_special_tokens', False),
-            padding_strategy=kwargs.get('padding_strategy', PaddingStrategy.DO_NOT_PAD),
-            truncation_strategy=kwargs.get('truncation_strategy', TruncationStrategy.DO_NOT_TRUNCATE),
-            max_length=kwargs.get('max_length'),
-            stride=kwargs.get('stride', 0),
-            pad_to_multiple_of=kwargs.get('pad_to_multiple_of'),
-            return_attention_mask=kwargs.get('return_attention_mask'),
-            return_token_type_ids=kwargs.get('return_token_type_ids'),
-            return_overflowing_tokens=kwargs.get('return_overflowing_tokens', False),
-            return_special_tokens_mask=kwargs.get('return_special_tokens_mask', False),
-            return_length=kwargs.get('return_length', False),
-            return_tensors=kwargs.get('return_tensors'),
-            verbose=kwargs.get('verbose', True),
-        )
-
-    def _save_pretrained(
-        self, save_directory: str | PathLike, file_names: Tuple[str], **kwargs
-    ) -> Tuple[str]:
-
-        return file_names

tokenizer.py

@@ -1,131 +1,129 @@
 # based on https://github.com/EleutherAI/gpt-neox/blob/main/megatron/tokenizer/tokenizer.py
-from abc import ABC
-import json
-import pathlib
+from __future__ import annotations
 
 import torch
-import tqdm
-from tokenizers import Tokenizer
-from transformers.tokenization_utils import PreTrainedTokenizer
-from abc import abstractmethod
-from typing import Any, List, Union
+
 import numpy as np
 
+from os import PathLike
+from typing import List, Tuple
 
-class HFAutoTokenizer:
-    def __init__(self, vocab_file):
-        self.tokenizer = Tokenizer.from_file(vocab_file)
-        self.eos = "</s>"
-        self.bos = "<s>"
-        self.eos_id = self.tokenize(self.eos)
-        self.bos_id = self.tokenize(self.bos)
-        self.vsize = 32000
-
-    def encode_to_list(self, text):
-        return self.tokenizer.encode(text, add_special_tokens=False)
-
-    def tokenize_file(self, input_file, output_file, verbose=False):
-        if verbose:
-            print(f"Tokenizing file: {input_file}")
-
-        if pathlib.Path(output_file).exists():
-            print(f"Output file {output_file} already exists, skipping")
-            return
-        with open(input_file, "r") as fin, open(output_file, "w") as fout:
-            for line in tqdm.tqdm(fin):
-                if verbose:
-                    print(f"Tokenizing line: {line[-200:]}")
-                data = json.loads(line.strip())
-                if "text" not in data.keys():
-                    break
-                tokenized_data = self.tokenize(data["text"])
-                fout.write(json.dumps({"tokens": tokenized_data}) + "\n")
-
-    def tokenize(self, text: str, *args, **kwargs):
-        ids = self.tokenizer.encode(text)
-        if type(ids) == list:
-            return torch.tensor(ids)
-        else:
-            return torch.tensor(ids.ids)
-
-    def tokenize_batch(self, text_batch):
-        return self.tokenizer.encode_batch(text_batch)
-
-    def detokenize(self, token_ids, skip_special_tokens=False):
-        return self.tokenizer.decode(token_ids, skip_special_tokens=skip_special_tokens)
-
-    def detokenize_batch(self, token_ids_batch, skip_special_tokens=False):
-        out = []
-        for token_ids in token_ids_batch:
-            out.append(
-                self.detokenize(
-                    [t.item() for t in token_ids],
-                    skip_special_tokens=skip_special_tokens,
-                )
-            )
-        return out
+from tokenizers import Tokenizer
+from transformers.tokenization_utils import PreTrainedTokenizer
+from transformers.tokenization_utils_base import BatchEncoding, TruncationStrategy
+from transformers.utils.generic import TensorType, PaddingStrategy
 
-    @property
-    def eod(self):
-        return self.eod_id
 
-    @property
-    def vocab_size(self):
-        return 32000
+EMPTY: str = ""
 
 
 class ByteTokenizer(PreTrainedTokenizer):
+
     """UTF-8 Encoder."""
-    def __init__(self):
-        super().__init__(
-            bos_token=self.decode_token(2),
-            eos_token=self.decode_token(0),
-            unk_token=self.decode_token(0),
-            pad_token=self.decode_token(1),
-            mask_token=self.decode_token(3),
-        )
-    
+
+    @classmethod
+    def from_pretrained(cls, model_id: str | PathLike, **kwargs) -> ByteTokenizer:
+
+        return cls(**kwargs, byte_level=True)
+
     @property
     def vocab_size(self) -> int:
+
         return 512
 
-    @classmethod
-    def from_pretrained(cls, *args, **kwargs):
-        return cls()
+    @property
+    def byte_level(self) -> bool:
+
+        return self.init_kwargs.get('byte_level', True)
+
+    def get_vocab(self) -> Dict[str, int]:
+
+        return {chr(i): i for i in range(self.vocab_size)}
 
-    def get_vocab(self):
-        return {str(i): i for i in range(512)}
+    def __len__(self) -> int:
+
+        return self.vocab_size
+
+    def clamp(self, n: int) -> int:
 
-    def clamp(self, n):
         return max(32, min(n, self.vocab_size))
 
-    def decode_token(self, token: int):
-        return str(chr(self.clamp(token)))
+    def _tokenize(self, text: str, **kwargs) -> List[str]:
+
+        return list(text)
 
-    def __call__(self, text: str, return_tensors: bool = False, *args, **kwargs):
-        ids = torch.tensor(self.tokenize(text), dtype=torch.long).unsqueeze(0)
-        return {"input_ids": ids} if return_tensors == False else ids
+    def byte_tokenize(self, text: str) -> np.ndarray:
 
-    def _tokenize(self, text: str):
         return np.frombuffer(text.encode('utf-8'), dtype=np.uint8)
-        
-    def tokenize(self, text: str):
-        return self._tokenize(text).tolist()
-
-    def tokenize_batch(self, text_batch: Union[List[str], str]):
-        if isinstance(text_batch, list):
-            return [self.tokenize(s) for s in text_batch]
-        else:
-            return self.tokenize(text_batch)
-
-    def decode(self, token_ids):
-        return "".join(list(map(self.decode_token, token_ids)))
-
-    def decode_batch(self, token_ids: Union[List[str], str]):
-        if isinstance(token_ids, list):
-            return [self.decode(s) for s in token_ids]
-        
-        elif isinstance(token_ids, torch.Tensor):
-            return [self.decode(s) for s in token_ids.tolist()]
-        else:
-            return self.decode(token_ids)
+
+    def _convert_token_to_id(self, token: str) -> int:
+
+        return self.clamp(ord(token))
+
+    def _convert_id_to_token(self, index: int) -> str:
+
+        return chr(self.clamp(index))
+
+    def convert_tokens_to_string(self, tokens: List[str]) -> str:
+
+        return EMPTY.join(tokens)
+
+    def _decode(self, token_ids: List[int], **kwargs) -> str:
+
+        indices = np.asarray(token_ids, dtype=np.uint8)
+
+        return (
+            indices.clip(min=32, max=self.vocab_size, out=indices)
+            .tobytes()
+            .decode('utf-8')
+        )
+
+    def _encode_plus(self, text: str, **kwargs) -> BatchEncoding:
+
+        first_ids = self.byte_tokenize(text).tolist()
+
+        return self.prepare_for_model(
+            first_ids,
+            pair_ids=None,
+            add_special_tokens=kwargs.get('add_special_tokens', False),
+            padding=kwargs.get('padding_strategy', PaddingStrategy.DO_NOT_PAD).value,
+            truncation=kwargs.get('truncation_strategy', TruncationStrategy.DO_NOT_TRUNCATE).value,
+            max_length=kwargs.get('max_length'),
+            stride=kwargs.get('stride', 0),
+            pad_to_multiple_of=kwargs.get('pad_to_multiple_of'),
+            return_tensors=kwargs.get('return_tensors'),
+            prepend_batch_axis=True,
+            return_attention_mask=kwargs.get('return_attention_mask'),
+            return_token_type_ids=kwargs.get('return_token_type_ids'),
+            return_overflowing_tokens=kwargs.get('return_overflowing_tokens', False),
+            return_special_tokens_mask=kwargs.get('return_special_tokens_mask', False),
+            return_length=kwargs.get('return_length', False),
+            verbose=kwargs.get('verbose', True),
+        )
+
+    def _batch_encode_plus(self, batch_text: List[str], **kwargs) -> BatchEncoding:
+
+        input_ids = [(self.byte_tokenize(text).tolist(), None) for text in batch_text]
+
+        return self._batch_prepare_for_model(
+            input_ids,
+            add_special_tokens=kwargs.get('add_special_tokens', False),
+            padding_strategy=kwargs.get('padding_strategy', PaddingStrategy.DO_NOT_PAD),
+            truncation_strategy=kwargs.get('truncation_strategy', TruncationStrategy.DO_NOT_TRUNCATE),
+            max_length=kwargs.get('max_length'),
+            stride=kwargs.get('stride', 0),
+            pad_to_multiple_of=kwargs.get('pad_to_multiple_of'),
+            return_attention_mask=kwargs.get('return_attention_mask'),
+            return_token_type_ids=kwargs.get('return_token_type_ids'),
+            return_overflowing_tokens=kwargs.get('return_overflowing_tokens', False),
+            return_special_tokens_mask=kwargs.get('return_special_tokens_mask', False),
+            return_length=kwargs.get('return_length', False),
+            return_tensors=kwargs.get('return_tensors'),
+            verbose=kwargs.get('verbose', True),
+        )
+
+    def _save_pretrained(
+        self, save_directory: str | PathLike, file_names: Tuple[str], **kwargs
+    ) -> Tuple[str]:
+
+        return file_names