Update app.py

app.py

@@ -1,13 +1,20 @@
+import spacy
 import gradio as gr
-from transformers import pipeline
+from transformers import pipeline, AutoTokenizer
 from pysentimiento.preprocessing import preprocess_tweet
 
-pl = pipeline("ner", tokenizer="cardiffnlp/twitter-roberta-base", model="Recognai/veganuary_ner", aggregation_strategy="first")
+nlp = spacy.load("en_core_web_sm")
+
+tokenizer = AutoTokenizer.from_pretrained("cardiffnlp/twitter-roberta-base", add_prefix_space=True, model_max_length=512)
+pl = pipeline("ner", tokenizer=tokenizer, model="Recognai/veganuary_ner", aggregation_strategy="first")
 
 def ner(text):
     text = preprocess_tweet(text)
-    predictions = pl(text)    
-    return "\n".join([pred["word"] for pred in predictions if pred["entity_group"] == "LABEL_1"])
+    doc = nlp(text)
+    text = " ".join([token.text for token in doc])
+    predictions = pl(text)
+    mentions = [pred["word"].strip() for pred in predictions if pred["entity_group"] == "LABEL_1"]   
+    return "\n".join(mentions)
 
 iface = gr.Interface(
     ner,