reset

SmilesPeTokenizer.py

@@ -1,16 +1,12 @@
-from tokenizers import Tokenizer
+import codecs
+from SmilesPE.tokenizer import *
 
-def bpe_tokenizer(smiles_string):
-    # Load the tokenizer from the saved file
-    tokenizer = Tokenizer.from_file("chembl_bpe_tokenizer.json")
+def smilespe_tokenizer(smiles_string):
 
-    # Tokenize the SMILES string
-    encoded_output = tokenizer.encode(smiles_string)
+    spe_vob = codecs.open('chembl_smiles_tokenizer30000.txt')
+    spe = SPE_Tokenizer(spe_vob)
 
-    # To get the tokenized output as text
-    tokens_text = encoded_output.tokens
+    tokenized = spe.tokenize(smiles_string)
+
+    return tokenized
 
-    # To get the corresponding token IDs
-    token_ids = encoded_output.ids
-    
-    return tokens_text, token_ids

app.py

@@ -6,7 +6,7 @@ iface = gr.Interface(
     inputs=[
         gr.Textbox(label="SMILES"),         
     ],
-    outputs=["text", "text"]
+    outputs="text"
 )
 
 iface.launch()