Spaces:
Runtime error
Runtime error
File size: 2,120 Bytes
b347aa0 |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 |
# from minbpe import BasicTokenizer, RegexTokenizer
# tokenizer = RegexTokenizer()
# tokenizer.load("first.model")
# text_to_encode = "मुझसे क्या होगा अब"
# encoded_text = tokenizer.encode(text_to_encode)
# print("Encoded:", encoded_text) # Output: [258, 100, 258, 97, 99]
# # Print the tokenized text
# print("Tokenized Text:", encoded_text)
# # Decode text
# decoded_text = tokenizer.decode(encoded_text)
# print("Decoded:", decoded_text) # Output: "aaabdaaabac"
import gradio as gr
from minbpe import BasicTokenizer, RegexTokenizer
# Initialize the tokenizer
tokenizer = RegexTokenizer()
tokenizer.load("first.model")
# Define the encoding function
def encode_text(text):
encoded_text = tokenizer.encode(text)
return str(encoded_text)
# Define the decoding function
def decode_text(encoded_text):
encoded_list = list(map(int, encoded_text.strip('[]').split(',')))
decoded_text = tokenizer.decode(encoded_list)
return decoded_text
# Define the Gradio interface
def gradio_app():
with gr.Blocks() as demo:
gr.Markdown("# Text Encoder and Decoder")
with gr.Row():
with gr.Column():
text_input = gr.Textbox(label="Text to Encode")
encoded_output = gr.Textbox(label="Encoded Text", interactive=False)
encode_button = gr.Button("Encode")
def encode_handler(text):
return encode_text(text)
encode_button.click(fn=encode_handler, inputs=text_input, outputs=encoded_output)
with gr.Column():
encoded_input = gr.Textbox(label="Encoded Text")
decoded_output = gr.Textbox(label="Decoded Text", interactive=False)
decode_button = gr.Button("Decode")
def decode_handler(encoded_text):
return decode_text(encoded_text)
decode_button.click(fn=decode_handler, inputs=encoded_input, outputs=decoded_output)
return demo
# Launch the app
if __name__ == "__main__":
app = gradio_app()
app.launch()
|