Update app.py

app.py

@@ -1,112 +1,124 @@
+from transformers import pipeline
+import torch
+import streamlit as st
 from textblob import TextBlob
 from vaderSentiment.vaderSentiment import SentimentIntensityAnalyzer
-from transformers import pipeline
-import pandas as pd
+import os
 import re
-import streamlit as st
-
+import pandas as pd
 
 def translate_text_blob(text):
     blob = TextBlob(text)
     return str(blob.translate(from_lang="pt", to="en"))
 
+def sentiment_vader(text):
 
-def translate_text_transformer(text, pipeline):
-    return pipeline(text)[0]["translation_text"]
-
-
-def sentiment_vader(sentence, vader_object):
+    vader_object = SentimentIntensityAnalyzer()
 
-    sentiment_dict = vader_object.polarity_scores(sentence)
+    sentiment_dict = vader_object.polarity_scores(text)
     negative = sentiment_dict['neg']
     neutral = sentiment_dict['neu']
     positive = sentiment_dict['pos']
     compound = sentiment_dict['compound']
 
-    if sentiment_dict['compound'] >= 0.05:
+    if sentiment_dict['compound'] >= 0.05 :
         overall_sentiment = "Positive"
 
-    elif sentiment_dict['compound'] <= - 0.05:
+    elif sentiment_dict['compound'] <= - 0.05 :
         overall_sentiment = "Negative"
 
-    else:
+    else :
         overall_sentiment = "Neutral"
-
+  
     return overall_sentiment.upper()
 
+def classify_by_company(text):
+    path = r"\Companies"
+
+    for filename in os.listdir(path):
+        with open(path + '\\' + filename, 'r') as f:
+            companies = [word[:-1] for word in f.readlines()]
+            companies = "|".join(companies)
+            companies = "/" + companies + "/gm"
+        
+        if re.search(companies, text):
+            return filename[:-4] + " - Infered by company name in text"
+
+    return ""
+
+def load_models(parameters_list):
+    
+    translation_map = {
+        #Translation PT to EN
+        "TextBlob"      : "TextBlob",
+        "M2M100"        : "facebook/m2m100_418M",
+        "OPUS"          : "Helsinki-NLP/opus-mt-mul-en",
+        "T5"            : "unicamp-dl/translation-pt-en-t5",
+        "mBART"         : "Narrativa/mbart-large-50-finetuned-opus-en-pt-translation",
+    }
+
+    sentiment_map = {
+        #Sentiment Analysis
+        "VADER"         : "VADER",
+        "FinBERT"       : "ProsusAI/finbert",
+        "DistilBERT"    : "distilbert-base-uncased-finetuned-sst-2-english",
+        "BERT"          : "nlptown/bert-base-multilingual-uncased-sentiment",
+    }
+
+    zeroshot_map = {
+        #Zeroshot Classification
+        "RoBERTa"       : "joeddav/xlm-roberta-large-xnli", 
+        "mDeBERTa"      : "MoritzLaurer/mDeBERTa-v3-base-mnli-xnli", 
+        "DistilroBERTa" : "cross-encoder/nli-distilroberta-base",
+    }
+
+    candidate_labels = [
+        "Industrial Goods",
+        "Communications",
+        "Cyclic Consumption",
+        "Non-cyclical Consumption",
+        "Financial",
+        "Basic Materials",
+        #"Others",
+        "Oil, Gas and Biofuels",
+        "Health",
+        #"Initial Sector",
+        "Information Technology",
+        "Public utility"
+    ]
+
+    device_num = 0 if torch.cuda.is_available() else -1
+
+    if parameters_list[0] == "TextBlob":
+        out_translation = translate_text_blob(parameters_list[3])
+    else:
+        translation    = pipeline("translation_pt_to_en",     model=translation_map[parameters_list[0]], tokenizer=translation_map[parameters_list[0]], device=device_num)
+        out_translation    = translation(parameters_list[3])[0]["translation_text"]
+    
+    if parameters_list[1] == "VADER":
+        out_sentiment = sentiment_vader(out_translation)
+    else:
+        sentiment      = pipeline("sentiment-analysis",       model=sentiment_map[parameters_list[1]],   tokenizer=sentiment_map[parameters_list[1]],   device=device_num)
+        out_sentiment      = sentiment(out_translation)[0]["label"].upper()
+
+    company_classification = classify_by_company(parameters_list[3].upper())
+
+    if company_classification:
+        out_classification = company_classification
+    else:
+        classification = pipeline("zero-shot-classification", model=zeroshot_map[parameters_list[2]],    tokenizer=zeroshot_map[parameters_list[2]],    device=device_num)
+        out_classification =  classification(out_translation, candidate_labels)["labels"][0] + " - Infered by {}".format(parameters_list[2])
 
-def sentiment_finbert(text, pipeline):
-    result = pipeline(text)[0]
-    return result["label"].upper()
-
-
-def theme_classification(text, text_classifier):
-        if re.search(r"/Biomm|Nortcquimica|Ourofino|Alliar|Dasa|Fleury|Hapvida|Ihpardini|Kora Saude|Mater Dei|Odontoprev|Oncoclinicas|Qualicorp |Rede D Or|Baumer|Lifemed|Blau|D1000vfarma|Dimed|Hypera|Pague Menos|Profarma|Raiadrogasil|Viveo|Drogaraia|Ultrafarma|Drogasil|Raia/gm", text):
-            return "Health"
-        elif re.search(r"/Aura 360|Bradespar|Cba|Csnmineracao|Litel|Litela|Mmx Miner|Vale|Ferbasa|Gerdau|Gerdau Met|Sid Nacional|Usiminas|Mangels Indl|Panatlantica|Tekno|Paranapanema|Braskem|Dexxos Par|Fer Heringer|Nutriplant|Vittia|Cristal|Unipar|Dexco|Eucatex|Klabin|Melhor Sp|Suzano Hold|Suzano|Irani|Metal Iguacu|Sansuy|Eternit|Haga|Portobello|Azevedo|Sondotecnica|Tecnosolo|Embraer|Fras-le|Marcopolo|Randon Part|Recrusul|Riosulense|Tupy|Wetzel S\/a|Schulz|Weg|Aco Altona|Aeris|Armac|Bardella|Inepar|Kepler Weber|Metalfrio|Mills|Nordon Met|Pratica|Romi|Metisa|Stara|Taurus Armas|Azul|Gol|All Norte|All Paulista|Fer C Atlant|Mrs Logist|Rumo|Hidrovias|Log-in|Trevisa|Jsl|Tegma|Autoban|Ccr Sa|Conc Raposo|Conc Rio Ter|Ecopistas|Ecorodovias|Ecovias|Rod Colinas|Rod Tiete|Rt Bandeiras|Triangulosol|Triunfo Part|Viaoeste|Gruairport|Hmobi S.a|Invepar|Porto Vm|Salus Infra|Santos Brp|Wilson Sons|Atmasa|Bbmlogistica|Dtcom-direct|Estapar|Flex|Gps|Priner|Sequoia Log|Valid|Embpar|Minasmaquina|Rodobens|Wlm Ind Com/gm", text):
-            return "Industrial Goods"
-        elif re.search(r"/Algar Telec|Brisanet|Desktop|Oi|Telebras|Telef Brasil|Tim|Unifique|Cinesystem|Eletromidia|Vivo|Claro|Tim/gm", text):
-            return "Communications"
-        elif re.search(r"/Alphaville|Const A Lind|Cr2Cury|Cyrela Realt|DirecionalEvenEztecGafisaHelborInter SaJhsf PartJoao FortesKallas|Lavvi|Melnick|Mitre Realty|Moura Dubeux|MrvPdg RealtPlanoeplano|Rni|Rossi ResidTecnisaTegra Incorp|Tenda|TrisulViverCedroCoteminasDohlerInd Cataguas|KarstenPettenatiSantanenseSpringsTekaTex RenauxTrack Field|AlpargatasCambuciGrendeneVulcabrasMundialTechnosVivara|WhirlpoolUnicasa|Hercules|Iochp-maxion|Metal Leve|Plascar Part|Hoteis Othon|Bk Brasil|Imc|Bic MonarkEstrelaSpturisTime For Fun|Cvc Brasil|Smart Fit|Anima|BahemaCogna On|Cruzeiro Edu|Ser Educa|Yduqs Part|LocalizaLocamerica|Maestroloc|Movida|UnidasVamos|Dotz Sa|Arezzo CoCea Modas|GrazziotinGrupo Soma|GuararapesLe Lis Blanc|Lojas Marisa|Lojas Renner|Allied|Magaz LuizaVia|Americanas|Espacolaser|Grupo Sbf|Le Biscuit|Petz|Quero-quero|Saraiva Livr|Saraiva|Burguer King/gm", text):
-            return "Cyclic Consumption"
-        elif re.search(r"/3tentos|Agribrasil|Agrogalaxy|Aliperti|Boa Safra|Brasilagro|Pomifrutas|Raizen|Slc Agricola|Terrasantapa|Jallesmachad|Raizen Energ|Sao Martinho|Carnes E Derivados|Brf Sa|Excelsior|Jbs|Marfrig|Minerva|Minupar|Camil|J.macedo|Josapar|M.diasbranco|Oderich|Ambev|Grupo Natura|Bombril|Assai|Carrefour Br|Grupo Mateus|P.acucar-cbd|Natura|Pão de Açúcar|Raizen|JBS|Carrefour/gm", text):
-            return "Non-cyclical Consumption"
-        elif re.search(r"/Abc Brasil|Alfa Holding|Alfa Invest|Amazonia|Banco Bmg|Banco Inter|Banco Pan|Banese|Banestes|Banpara|Banrisul|Br Partners|Bradesco|Brasil|Brb Banco|Btgp Banco|Itausa|Itauunibanco|Merc Brasil|Merc Invest|Modalmais|Nord Brasil|Nu-nubank|Parana|Pine|Santander Br|Alfa Financ|Finansinos|Merc Financ|Bradesco Lsg|Dibens Lsg|Brazil Realt|Brazilian Sc|Brpr 56 Sec|Cibrasec|Eco Sec Agro|Gaia Agro|Gaia Securit|Octante Sec|Pdg Securit|Polo Cap Sec|Rbcapitalres|Truesec|Vertciasec|Wtorre Pic|Bndespar|Brazilian Fr|G2d Invest|Gp Invest|Ppla|B3|Boa Vista|Cielo|Clearsale|Csu Cardsyst|Getnet Br|Stone Co|Xp Inc|Alfa Consorc|Bbseguridade|Caixa Seguri|Porto Seguro|Seg Al Bahia|Sul America|Irbbrasil Re|Alper|Wiz|Alianscsonae|Br Malls Par|Br Propert|Cor Ribeiro|Cyre Com-ccp|Generalshopp|Habitasul|Hbr Realty|Igb|Iguatemi|Log Com Prop|Menezes Cort|Multiplan|Sao Carlos|Syn Prop Tec|Lopes Brasil|Nexpe|Mont Aranha|Par Al Bahia|Simpar|Cepac - Ctba|Cepac - Mcrj|Cepac - Pmsp|Inter|Pan|Banco do Brasil|BTG|Itaú|Nubank|Santander|Alfa|Merc|BNDES|/gm", text):
-            return "Financial"
-        elif re.search(r"/Aura 360|Bradespar|CBA|Csnmineracao|Litel|Litela|Mmx Miner|Vale|Ferbasa|Gerdau|Gerdau Met|Sid Nacional|Usiminas|Mangels Indl|Panatlantica|Tekno|Paranapanema|Braskem|Dexxos Par|Fer Heringer|Nutriplant|Vittia|Cristal|Unipar|Dexco|Eucatex|Klabin|Melhor Sp|Suzano Hold|Suzano|Irani|Metal Iguacu|Sansuy/gm", text):
-            return "Basic Materials"
-        #elif re.search(r"/524 Particip|Atompar|B Tech Eqi|Betapart |Cabinda Part|Caconde Part|Cemepe |Cims |Gama Part |Invest Bemge|Polpar |Prompt Part |Sudeste|Sul 116 Part|Ybyra/gm", text):
-            #return "Others"
-        elif re.search(r"/3r Petroleum|Cosan|Dommo|Enauta Part|Pet Manguinh|Petrobras|Petrorecsa|Petrorio|Ultrapar|Vibra|Lupatech|Oceanpact|Osx Brasil/gm", text):
-            return "Oil, Gas and Biofuels"
-        elif re.search(r"/Intelbras|Multilaser|Positivo Tec|Bemobi Tech|Brq|Enjoei|Getninjas|Infracomm|Locaweb|Meliuz|Mobly|Neogrid|Padtec|Quality Soft|Sinqia|Tc|Totvs |Wdc Networks|Westwing/gm", text):
-            return "Information Technology"
-        elif re.search(r"/Aes Brasil|Aes Sul|Aesoperacoes|Afluente T|Alupar|Ampla Energ|Auren|Cachoeira|Ceb|Ceee-d|Ceee-t|Celesc|Celgpar|Celpe|Cemig|Cemig Dist|Cemig Gt|Coelba|Coelce|Comerc Par|Copel|Cosern|Cpfl Energia|Cpfl Geracao|Cpfl Piratin|Cpfl Renovav|Ebe|Elektro|Eletrobras|Eletropar|Emae|Energias Br|Energisa|Energisa Mt|Enersul|Eneva|Engie Brasil|Eqtl Para|Eqtlmaranhao|Equatorial|Escelsa|Fgenergia|Ger Paranap|Itapebi|Light|Light S/a|Neoenergia|Omegaenergia|Paul F Luz|Proman|Rede Energia|Renova|Statkraft|Sto Antonio|Taesa|Term. Pe Iii|Termope|Tran Paulist|Uptick|Ambipar|Casan|Copasa|Igua Sa|Orizon|Sabesp|Sanepar|Sanesalto|Ceg|Comgas|Compass|AES|Cemig|CEMIG/gm", text):
-            return "Public utility"
-        else:
-            labels = [  
-                      "Industrial Goods",
-                      "Communications",
-                      "Cyclic Consumption",
-                      "Non-cyclical Consumption",
-                      "Financial",
-                      "Basic Materials",
-                      #"Others",
-                      "Oil, Gas and Biofuels",
-                      "Health",
-                      #"Initial Sector",
-                      "Information Technology",
-                      "Public utility"
-                     ]
-
-            template = "The economic sector of this set of words is {}."
-
-            results = text_classifier(text, labels, hypothesis_template=template)
-
-            index = results["scores"].index(max(results["scores"]))
-
-            return str(results["labels"][index])
-
-m2m100 = pipeline("translation_pt_to_en", src_lang="pt", tgt_lang="en", model="models/m2m100")
-opus = pipeline("translation_pt_to_en", src_lang="pt", tgt_lang="en", model="models/opus")
-finbert = pipeline("sentiment-analysis",model="models/finbert")
-classifier = pipeline("zero-shot-classification", model="models/roberta")
-sid_obj = SentimentIntensityAnalyzer()
-
-
-sheet_id = "1IGFSKnnmQndKVmGOWMCbsvJJMU_2jvnm"
-sheet_name = "Sheet1"
-url = f"https://docs.google.com/spreadsheets/d/{sheet_id}/gviz/tq?tqx=out:csv&sheet={sheet_name}"
-df = pd.read_csv(url)
+    return out_translation, out_sentiment, out_classification
+
+df = pd.read_csv(r"\Data\Hugging Face_DF.csv")
 
 header = st.container()
 model = st.container()
 model_1, model_2 = st.columns(2)
 dataset = st.container()
 analysis = st.container()
-
-with st.sidebar:
-    st.markdown("# Lorem Ipsum\nLorem ipsum dolor sit amet, consectetur adipiscing elit. Praesent sapien tortor, suscipit quis ornare ut, laoreet vitae nisi. Mauris quis consectetur risus, non blandit mauris. Sed ut odio tempor, ullamcorper leo eu, mollis eros. Curabitur pretium sollicitudin sapien, vel mattis augue convallis quis. Suspendisse eleifend turpis non nunc gravida, aliquet hendrerit orci viverra. Sed aliquet, nunc eu posuere tempor, libero ex dignissim velit, ut ultricies erat felis at urna. Proin metus augue, commodo in faucibus sed, aliquet ac eros. Nullam turpis leo, dictum eu tellus a, aliquam egestas velit. Suspendisse cursus augue a sem dapibus, eu faucibus nisl ultrices. Cras tortor ipsum, luctus vitae tincidunt id, dapibus id justo. Sed mi nunc, tempor eu iaculis in, tristique cursus massa. Integer metus felis, pulvinar ut aliquam ut, consectetur in nulla.")
+analysis_1, analysis_2 = st.columns(2)
 
 with header:
     st.title("IC 2022 Classificação de Dados Financeiros")
@@ -117,10 +129,9 @@ with model:
     st.header("Modelo para Tradução e Classificação!")
 
     with model_1:
-        translator = st.selectbox(
-            'Qual modelo você deseja usar para tradução?', ('TextBlob', 'M2M100', 'OPUS'))
-        sentimentor = st.selectbox(
-            'Qual modelo você deseja usar para a análise de sentimentos?', ('Vader', 'FinBERT'))
+        translation_pt_to_en      = st.selectbox('Qual modelo você deseja usar para tradução?', ('TextBlob', 'M2M100', 'OPUS', 'T5', 'mBART'))
+        sentiment_analysis        = st.selectbox('Qual modelo você deseja usar para análise de sentimento?', ('VADER', 'FinBERT', 'DistilBERT', 'BERT'))
+        zero_shot_classification  = st.selectbox('Qual modelo você deseja usar para classificação?', ('RoBERTa', 'mDeBERTa', 'DistilroBERTa'))
 
         text = st.text_input(label="Coloque seu texto sobre mercado financeiro em português!", value=r"As ações da Raia Drogasil subiram em 98% desde o último bimestre, segundo as avaliações da revista!")
         
@@ -128,40 +139,13 @@ with model:
 
     with model_2:
         if submit:
-            if translator == "TextBlob":
-                text_en = translate_text_blob(text)
-            elif translator == "M2M100":
-                text_en = translate_text_transformer(text, m2m100)
-            elif translator == "OPUS":
-                text_en = translate_text_transformer(text, opus)
-
-            if sentimentor == "Vader":
-                sentiment = sentiment_vader(text_en, sid_obj)
-            elif sentimentor == "FinBERT":
-                sentiment = sentiment_finbert(text_en, finbert)
-            
-            theme = theme_classification(text_en, classifier)
-
-            st.text_area(label="Translation:", value=text_en, height=150)
-            st.text_area(label="Sentiment:", value=sentiment, height=150)
-            st.text_area(label="Theme:", value=theme, height=150)
-
-'''
-with model:
-    st.header("Modelo para Tradução e Classificação!")
-    text = st.text_area(
-        "Coloque seu texto sobre mercado financeiro em português!")
-
-    submit = st.button('Generate')
+            with st.spinner('Wait for it...'):
+                parameters = [translation_pt_to_en, sentiment_analysis, zero_shot_classification, text]
+                outputs = load_models(parameters)
 
-    if text:
-        text_en = translate_text(text)
-        st.write("*Translation:* {}".format(text_en))
-        sentiment = sentiment_classification(text_en, sid_obj)
-        st.write("*Sentiment:* {} - {}".format(sentiment[0], sentiment[1]))
-        theme = theme_classification(text_en, classifier)
-        st.write("*Theme:* {}".format(theme))
-'''
+            st.write("Translation..................................................................: \n {} \n \n".format(outputs[0]))
+            st.write("Sentiment...................................................................: \n {} \n \n".format(outputs[1]))
+            st.write("Classification...............................................................: \n {} \n \n".format(outputs[2]))
 
 with dataset:
     st.header("Dados utilizados no projeto!")
@@ -175,4 +159,7 @@ with dataset:
     st.write("\t*- Texts:* Coluna que mostra os textos financeiros")
 
 with analysis:
-    st.header("Visualização dos dados utilizados")
+    st.header("Visualização dos dados utilizados através de WordClouds")
+
+    with analysis_1:
+        pass