New tokenizer

tokens.py

@@ -3,11 +3,11 @@ from datasets import load_dataset
 from tokenizers import ByteLevelBPETokenizer
 
 # Load dataset
-dataset = load_dataset("oscar", "unshuffled_deduplicated_es", split="train")
+dataset = load_dataset("oscar", "unshuffled_deduplicated_es", split="train[:5000000]")
 
 # Instantiate tokenizer
 tokenizer = ByteLevelBPETokenizer()
-def batch_iterator(batch_size=1_000_000):
+def batch_iterator(batch_size=100_000):
     for i in range(0, len(dataset), batch_size):
         yield dataset["text"][i: i + batch_size]