Update app.py

app.py

@@ -1,19 +1,27 @@
 from textblob import TextBlob
 from vaderSentiment.vaderSentiment import SentimentIntensityAnalyzer
-from transformers import pipeline
+from transformers import pipeline, M2M100ForConditionalGeneration, M2M100Tokenizer, MBart50TokenizerFast, MBartForConditionalGeneration, AutoTokenizer, AutoModelForSequenceClassification
 import pandas as pd
 import re
 import streamlit as st
-from datasets import load_dataset
 
-def translate_text(text):
+def translate_text_blob(text):
     blob = TextBlob(text)
-    try:
-       return str(blob.translate(from_lang="pt", to="en"))
-    except:
-       return text
+    return str(blob.translate(from_lang="pt", to="en"))
 
-def sentiment_classification(sentence, vader_object):
+def translate_text_M2M100(text, model, tokenizer):
+    encoded = tokenizer(text, return_tensors="pt")
+    generated_tokens = model.generate(**encoded, forced_bos_token_id=tokenizer.get_lang_id("en"))
+    return tokenizer.batch_decode(generated_tokens, skip_special_tokens=True)[0]
+
+def translate_BART(text, model, tokenizer):
+    inputs = tokenizer(text, return_tensors='pt')
+    input_ids = inputs.input_ids
+    attention_mask = inputs.attention_mask
+    output = model.generate(input_ids, attention_mask=attention_mask, forced_bos_token_id=tokenizer.lang_code_to_id['en_XX'])
+    return tokenizer.decode(output[0], skip_special_tokens=True)
+
+def sentiment_vader(sentence, vader_object):
 
     sentiment_dict = vader_object.polarity_scores(sentence)
     negative = sentiment_dict['neg']
@@ -30,56 +38,26 @@ def sentiment_classification(sentence, vader_object):
     else :
         overall_sentiment = "Neutral"
   
-    return overall_sentiment, sentiment_dict['compound']
- 
-def theme_classification(text, text_classifier):
-        if re.search(r"/Biomm|Nortcquimica|Ourofino|Alliar|Dasa|Fleury|Hapvida|Ihpardini|Kora Saude|Mater Dei|Odontoprev|Oncoclinicas|Qualicorp |Rede D Or|Baumer|Lifemed|Blau|D1000vfarma|Dimed|Hypera|Pague Menos|Profarma|Raiadrogasil|Viveo|Drogaraia|Ultrafarma|Drogasil|Raia/gm", text):
-            return "Health"
-        elif re.search(r"/Aura 360|Bradespar|Cba|Csnmineracao|Litel|Litela|Mmx Miner|Vale|Ferbasa|Gerdau|Gerdau Met|Sid Nacional|Usiminas|Mangels Indl|Panatlantica|Tekno|Paranapanema|Braskem|Dexxos Par|Fer Heringer|Nutriplant|Vittia|Cristal|Unipar|Dexco|Eucatex|Klabin|Melhor Sp|Suzano Hold|Suzano|Irani|Metal Iguacu|Sansuy|Eternit|Haga|Portobello|Azevedo|Sondotecnica|Tecnosolo|Embraer|Fras-le|Marcopolo|Randon Part|Recrusul|Riosulense|Tupy|Wetzel S\/a|Schulz|Weg|Aco Altona|Aeris|Armac|Bardella|Inepar|Kepler Weber|Metalfrio|Mills|Nordon Met|Pratica|Romi|Metisa|Stara|Taurus Armas|Azul|Gol|All Norte|All Paulista|Fer C Atlant|Mrs Logist|Rumo|Hidrovias|Log-in|Trevisa|Jsl|Tegma|Autoban|Ccr Sa|Conc Raposo|Conc Rio Ter|Ecopistas|Ecorodovias|Ecovias|Rod Colinas|Rod Tiete|Rt Bandeiras|Triangulosol|Triunfo Part|Viaoeste|Gruairport|Hmobi S.a|Invepar|Porto Vm|Salus Infra|Santos Brp|Wilson Sons|Atmasa|Bbmlogistica|Dtcom-direct|Estapar|Flex|Gps|Priner|Sequoia Log|Valid|Embpar|Minasmaquina|Rodobens|Wlm Ind Com/gm", text):
-            return "Industrial Goods"
-        elif re.search(r"/Algar Telec|Brisanet|Desktop|Oi|Telebras|Telef Brasil|Tim|Unifique|Cinesystem|Eletromidia|Vivo|Claro|Tim/gm", text):
-            return "Communications"
-        elif re.search(r"/Alphaville|Const A Lind|Cr2Cury|Cyrela Realt|DirecionalEvenEztecGafisaHelborInter SaJhsf PartJoao FortesKallas|Lavvi|Melnick|Mitre Realty|Moura Dubeux|MrvPdg RealtPlanoeplano|Rni|Rossi ResidTecnisaTegra Incorp|Tenda|TrisulViverCedroCoteminasDohlerInd Cataguas|KarstenPettenatiSantanenseSpringsTekaTex RenauxTrack Field|AlpargatasCambuciGrendeneVulcabrasMundialTechnosVivara|WhirlpoolUnicasa|Hercules|Iochp-maxion|Metal Leve|Plascar Part|Hoteis Othon|Bk Brasil|Imc|Bic MonarkEstrelaSpturisTime For Fun|Cvc Brasil|Smart Fit|Anima|BahemaCogna On|Cruzeiro Edu|Ser Educa|Yduqs Part|LocalizaLocamerica|Maestroloc|Movida|UnidasVamos|Dotz Sa|Arezzo CoCea Modas|GrazziotinGrupo Soma|GuararapesLe Lis Blanc|Lojas Marisa|Lojas Renner|Allied|Magaz LuizaVia|Americanas|Espacolaser|Grupo Sbf|Le Biscuit|Petz|Quero-quero|Saraiva Livr|Saraiva|Burguer King/gm", text):
-            return "Cyclic Consumption"
-        elif re.search(r"/3tentos|Agribrasil|Agrogalaxy|Aliperti|Boa Safra|Brasilagro|Pomifrutas|Raizen|Slc Agricola|Terrasantapa|Jallesmachad|Raizen Energ|Sao Martinho|Carnes E Derivados|Brf Sa|Excelsior|Jbs|Marfrig|Minerva|Minupar|Camil|J.macedo|Josapar|M.diasbranco|Oderich|Ambev|Grupo Natura|Bombril|Assai|Carrefour Br|Grupo Mateus|P.acucar-cbd|Natura|Pão de Açúcar|Raizen|JBS|Carrefour/gm", text):
-            return "Non-cyclical Consumption"
-        elif re.search(r"/Abc Brasil|Alfa Holding|Alfa Invest|Amazonia|Banco Bmg|Banco Inter|Banco Pan|Banese|Banestes|Banpara|Banrisul|Br Partners|Bradesco|Brasil|Brb Banco|Btgp Banco|Itausa|Itauunibanco|Merc Brasil|Merc Invest|Modalmais|Nord Brasil|Nu-nubank|Parana|Pine|Santander Br|Alfa Financ|Finansinos|Merc Financ|Bradesco Lsg|Dibens Lsg|Brazil Realt|Brazilian Sc|Brpr 56 Sec|Cibrasec|Eco Sec Agro|Gaia Agro|Gaia Securit|Octante Sec|Pdg Securit|Polo Cap Sec|Rbcapitalres|Truesec|Vertciasec|Wtorre Pic|Bndespar|Brazilian Fr|G2d Invest|Gp Invest|Ppla|B3|Boa Vista|Cielo|Clearsale|Csu Cardsyst|Getnet Br|Stone Co|Xp Inc|Alfa Consorc|Bbseguridade|Caixa Seguri|Porto Seguro|Seg Al Bahia|Sul America|Irbbrasil Re|Alper|Wiz|Alianscsonae|Br Malls Par|Br Propert|Cor Ribeiro|Cyre Com-ccp|Generalshopp|Habitasul|Hbr Realty|Igb|Iguatemi|Log Com Prop|Menezes Cort|Multiplan|Sao Carlos|Syn Prop Tec|Lopes Brasil|Nexpe|Mont Aranha|Par Al Bahia|Simpar|Cepac - Ctba|Cepac - Mcrj|Cepac - Pmsp|Inter|Pan|Banco do Brasil|BTG|Itaú|Nubank|Santander|Alfa|Merc|BNDES|/gm", text):
-            return "Financial"
-        elif re.search(r"/Aura 360|Bradespar|CBA|Csnmineracao|Litel|Litela|Mmx Miner|Vale|Ferbasa|Gerdau|Gerdau Met|Sid Nacional|Usiminas|Mangels Indl|Panatlantica|Tekno|Paranapanema|Braskem|Dexxos Par|Fer Heringer|Nutriplant|Vittia|Cristal|Unipar|Dexco|Eucatex|Klabin|Melhor Sp|Suzano Hold|Suzano|Irani|Metal Iguacu|Sansuy/gm", text):
-            return "Basic Materials"
-        #elif re.search(r"/524 Particip|Atompar|B Tech Eqi|Betapart |Cabinda Part|Caconde Part|Cemepe |Cims |Gama Part |Invest Bemge|Polpar |Prompt Part |Sudeste|Sul 116 Part|Ybyra/gm", text):
-            #return "Others"
-        elif re.search(r"/3r Petroleum|Cosan|Dommo|Enauta Part|Pet Manguinh|Petrobras|Petrorecsa|Petrorio|Ultrapar|Vibra|Lupatech|Oceanpact|Osx Brasil/gm", text):
-            return "Oil, Gas and Biofuels"
-        elif re.search(r"/Intelbras|Multilaser|Positivo Tec|Bemobi Tech|Brq|Enjoei|Getninjas|Infracomm|Locaweb|Meliuz|Mobly|Neogrid|Padtec|Quality Soft|Sinqia|Tc|Totvs |Wdc Networks|Westwing/gm", text):
-            return "Information Technology"
-        elif re.search(r"/Aes Brasil|Aes Sul|Aesoperacoes|Afluente T|Alupar|Ampla Energ|Auren|Cachoeira|Ceb|Ceee-d|Ceee-t|Celesc|Celgpar|Celpe|Cemig|Cemig Dist|Cemig Gt|Coelba|Coelce|Comerc Par|Copel|Cosern|Cpfl Energia|Cpfl Geracao|Cpfl Piratin|Cpfl Renovav|Ebe|Elektro|Eletrobras|Eletropar|Emae|Energias Br|Energisa|Energisa Mt|Enersul|Eneva|Engie Brasil|Eqtl Para|Eqtlmaranhao|Equatorial|Escelsa|Fgenergia|Ger Paranap|Itapebi|Light|Light S/a|Neoenergia|Omegaenergia|Paul F Luz|Proman|Rede Energia|Renova|Statkraft|Sto Antonio|Taesa|Term. Pe Iii|Termope|Tran Paulist|Uptick|Ambipar|Casan|Copasa|Igua Sa|Orizon|Sabesp|Sanepar|Sanesalto|Ceg|Comgas|Compass|AES|Cemig|CEMIG/gm", text):
-            return "Public utility"
-        else:
-            labels = [  
-                      "Industrial Goods",
-                      "Communications",
-                      "Cyclic Consumption",
-                      "Non-cyclical Consumption",
-                      "Financial",
-                      "Basic Materials",
-                      #"Others",
-                      "Oil, Gas and Biofuels",
-                      "Health",
-                      #"Initial Sector",
-                      "Information Technology",
-                      "Public utility"
-                     ]
-
-            template = "The economic sector of this set of words is {}."
-
-            results = text_classifier(text, labels, hypothesis_template=template)
-
-            index = results["scores"].index(max(results["scores"]))
-
-            return str(results["labels"][index])
+    return overall_sentiment.upper()
+
+def sentiment_finbert(text, pipeline):
+    result = pipeline(text)[0]
+    return result["label"].upper()
+
+m2m100 = M2M100ForConditionalGeneration.from_pretrained("facebook/m2m100_418M")
+m2m100_token = M2M100Tokenizer.from_pretrained("facebook/m2m100_418M")
+m2m100_token.src_lang = "pt"
+
+BART_token = MBart50TokenizerFast.from_pretrained('Narrativa/mbart-large-50-finetuned-opus-pt-en-translation')
+BART = MBartForConditionalGeneration.from_pretrained('Narrativa/mbart-large-50-finetuned-opus-pt-en-translation')
+BART_token.src_lang = 'pt_XX'
+
+finbert_token = AutoTokenizer.from_pretrained("ProsusAI/finbert")
+finbert = AutoModelForSequenceClassification.from_pretrained("ProsusAI/finbert", num_labels=3)
+finbert_pipeline = pipeline("sentiment-analysis", model=finbert, tokenizer=finbert_token)
 
 sid_obj = SentimentIntensityAnalyzer()
+
 classifier = pipeline("zero-shot-classification", model="joeddav/xlm-roberta-large-xnli")
 
 sheet_id = "1IGFSKnnmQndKVmGOWMCbsvJJMU_2jvnm"
@@ -91,6 +69,7 @@ header = st.container()
 model = st.container()
 dataset = st.container()
 analysis = st.container()
+model_1, model_2 = st.columns(2)
 
 with st.sidebar:
    st.markdown("# Lorem Ipsum\nLorem ipsum dolor sit amet, consectetur adipiscing elit. Praesent sapien tortor, suscipit quis ornare ut, laoreet vitae nisi. Mauris quis consectetur risus, non blandit mauris. Sed ut odio tempor, ullamcorper leo eu, mollis eros. Curabitur pretium sollicitudin sapien, vel mattis augue convallis quis. Suspendisse eleifend turpis non nunc gravida, aliquet hendrerit orci viverra. Sed aliquet, nunc eu posuere tempor, libero ex dignissim velit, ut ultricies erat felis at urna. Proin metus augue, commodo in faucibus sed, aliquet ac eros. Nullam turpis leo, dictum eu tellus a, aliquam egestas velit. Suspendisse cursus augue a sem dapibus, eu faucibus nisl ultrices. Cras tortor ipsum, luctus vitae tincidunt id, dapibus id justo. Sed mi nunc, tempor eu iaculis in, tristique cursus massa. Integer metus felis, pulvinar ut aliquam ut, consectetur in nulla.")
@@ -99,13 +78,42 @@ with header:
    st.title("IC 2022 Classificação de Dados Financeiros")
    st.write("Lorem ipsum dolor sit amet, consectetur adipiscing elit. Praesent sapien tortor, suscipit quis ornare ut, laoreet vitae nisi. Mauris quis consectetur risus, non blandit mauris. Sed ut odio tempor, ullamcorper leo eu, mollis eros.")
 
+with model_1:
+   st.header("Modelo para Tradução e Classificação!")
+   translator  = st.selectbox('Qual modelo você deseja usar para tradução?', ('TextBlob', 'M2M100', 'BART'))
+   sentimentor = st.selectbox('Qual modelo você deseja usar para a análise de sentimentos?', ('Vader', 'FinBERT'))
+   text = st.text_area("Coloque seu texto sobre mercado financeiro em português!", "As ações da Ultrafarma subiram em 98% no último bimestre, segundo os dados da revista!")
+   submit = st.button('Gerar!')
+
+with model_2:
+   if submit:
+        with st.spinner('Aguarde enquanto estamos processando...'):
+            if translator == "TextBlob":
+                text_en = translate_text_blob(text)
+            elif translator == "M2M100":
+                text_en = translate_text_M2M100(text, m2m100, m2m100_token)
+            elif translator == "BART":
+                text_en = translate_text_M2M100(text, BART, BART_token)
+
+            if sentimentor == "Vader":
+                sentiment = sentiment_vader(text_en, sid_obj)
+            elif sentimentor == "FinBERT":
+                sentiment = sentiment_finbert(text_en, finbert_pipeline)
+    
+    st.write(text_en)
+    st.write(sentiment)
+   
+   
+   
 with model:
    st.header("Modelo para Tradução e Classificação!")
    text = st.text_area("Coloque seu texto sobre mercado financeiro em português!")
    
+   submit = st.button('Generate')
+   
    if text:
-      text_en = translate_text(text)
-      st.write("*Translation:* {}".format(text_en))
+        text_en = translate_text(text)
+        st.write("*Translation:* {}".format(text_en))
       sentiment = sentiment_classification(text_en, sid_obj)
       st.write("*Sentiment:* {} - {}".format(sentiment[0], sentiment[1]))
       theme = theme_classification(text_en, classifier)