<jupyter_start><jupyter_text>**Download do dataset original:**https://www.kaggle.com/luisfredgs/imdb-ptbr/kernels**Dados usados nesse tutorial:**https://drive.google.com/drive/folders/1hwhqN-CUIGGZZXpVnka08yaItZsMQapx?usp=sharing  Processamento e análise dos dados<jupyter_code># Preparando ambiente (importando bibliotecas e downloads...)

!pip install nltk
import nltk
nltk.download('rslp')
nltk.download('stopwords')
nltk.download('punkt')
import re
import pandas as pd 
# Importando dataset
df = pd.read_csv("dataset.csv", sep="," , encoding="utf8")
# Separacao dos dados por sentimento
df.groupby('sentiment').count()<jupyter_output><empty_output><jupyter_text>Removendo e tratando dados<jupyter_code># Remove columns e create column
df.drop(columns=['id', 'text_en'], axis=1, inplace=True)
df['classification'] = df["sentiment"].replace(["neg", "pos"],[0, 1])

# Texto para minusculo
text_lower = [t.lower() for t in df['text_pt']]
df['text_pt'] = text_lower

df.head(5)
# funcao para remover brackets
def remove_brackets(column):
    for x in range(1,len(column)):
        return(re.sub('[\[\]]','',repr(column)))<jupyter_output><empty_output><jupyter_text>Stemmer e stopwords<jupyter_code>%%time

from nltk.tokenize import word_tokenize 
stop_words = nltk.corpus.stopwords.words('portuguese')
stemmer = nltk.stem.RSLPStemmer()

# Trabalhar com stemmer e stopwords da base de treinamento/teste

for x in range(0,len(df['text_pt'])):

    # Remover as stop words do texto
    word_tokens = word_tokenize(df['text_pt'][x]) 
    filtered_sentence = [w for w in word_tokens if not w in stop_words] 
    
    # Remover sufixos 
    line=[]
    text_tokenized = word_tokenize((remove_brackets(filtered_sentence)))
    line =  [stemmer.stem(word) for word in text_tokenized]
    df['text_pt'][x] = (remove_brackets(line))


from sklearn.feature_extraction.text import CountVectorizer
from nltk.tokenize import RegexpTokenizer

# Regex para remover alguns valores do dataset  (simbolos, numeros...)
token = RegexpTokenizer(r'[a-zA-Z0-9]+')

# Cria o 'vetorizador' de acordo com os parametros abaixo
cv = CountVectorizer(lowercase=True,stop_words=None,ngram_range = (1,2),
                     tokenizer = token.tokenize)

# Matrixsparse da representação da coluna  text_pt
text_counts= cv.fit_transform(df['text_pt'])

# Vocabulario
cv.vocabulary_<jupyter_output><empty_output><jupyter_text>Testando o modelo<jupyter_code># Importando biliotecas para selecao de amostra, modelo e avaliação do modelo.
from sklearn.model_selection import train_test_split
from sklearn.naive_bayes import MultinomialNB
from sklearn import metrics

# Divindo no dataset em treino e teste
X_train, X_test, y_train, y_test = train_test_split(text_counts, df['classification'], 
                                                    test_size=0.34, random_state=1, 
                                                    shuffle=True)
# Criar modelo e treinar
clf = MultinomialNB().fit(X_train, y_train)

# Fazendo  predict do valor de X para teste de acuracidade
y_predicted= clf.predict(X_test)
print("MultinomialNB Accuracy:",metrics.accuracy_score(y_test, y_predicted).round(3))<jupyter_output>MultinomialNB Accuracy: 0.912<jupyter_text>Recebendo dados do texto (txt) para gerar grafíco<jupyter_code># Separa por paragrafos 
with open('./texto_teste.txt', 'r') as file_teste:
    paragraph = file_teste.read().split('\n\n')

# Separa por frases
with open('./texto_teste.txt', 'r') as file_teste:
    phrase = file_teste.read().split('.')<jupyter_output><empty_output><jupyter_text>Fazer stemmer e stopwords do conteúdo do texto<jupyter_code>#Importar stemmer novamente
stemmer = nltk.stem.RSLPStemmer()

# Criar dataframe
df_result = pd.DataFrame()


# Fazer a tokanização, remocao de stop words e 
# transformar os dados para predict
neg,pos=0,0
for x in range(0,len(phrase)-1):

    # Texto tokenizado
    text_tokenized = word_tokenize(phrase[x])

    # Remove stop words do texto
    filtered_sentence = [w for w in text_tokenized if not w in stop_words] 

    # Cria stemmer do texto input
    line =  [stemmer.stem(word) for word in filtered_sentence]
    line = (remove_brackets(line))

    # Criar prediction para cada frase
    value_trans = cv.transform([line])
    predict_phrase = clf.predict(value_trans)

    # Contar por tipo de prediction (positivo e negativo)
    if predict_phrase==0:pos+=1
    else:neg+=1

# Salvar valores no dataframe
df_result['positive'] = [pos]
df_result['negative'] = [neg]<jupyter_output>CPU times: user 135 ms, sys: 0 ns, total: 135 ms
Wall time: 135 ms<jupyter_text>Criando gráfico de análise de sentimento<jupyter_code>def generate_piechart(df_result):
    
    import matplotlib.pyplot as plt
    labels = df_result.columns.tolist()
    sizes = df_result.values.tolist()[0]
    color = ['lightskyblue', 'lightcoral']
    explode = (0.15, 0)

    fig1, ax1 = plt.subplots(figsize=(5,5))
    ax1.pie(sizes, labels=labels,  explode=explode,
            shadow=True, autopct='%1.1f%%',  startangle=140, colors=color)

    ax1.set_title('Sentiment Analysis by phrases - NLTK', fontsize=15)

    ax1.axis('equal')
    plt.show()
    print("Quantity by paragraph: {}".format(len(paragraph)))
    print("Quantity by phrases: {}".format(len(phrase)-1))
    print("Quantity by positives phrases: {}".format(df_result['positive']
                                                     .values.tolist()[0]))
    
    print("Quantity by negatives phrases: {}".format(df_result['negative']
                                                     .values.tolist()[0]))
    
# Gerar gráfico    
generate_piechart(df_result)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.11814644499469402, 'fraction_numerical': 0.010965688008489564, 'mean_word_length': 4.236111111111111, 'pattern_counts': {'":': 0, '<': 27, '<?xml version=': 0, '>': 27, 'https://': 2, 'lorem ipsum': 0, 'www.': 1, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 13, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '7', 'dir': 'jupyter-structured-clean-dedup', 'id': '491789', 'license': 'MIT', 'n_tokens_mistral': 2050, 'n_tokens_neox': 1961, 'n_words': 442, 'path': 'NPL.ipynb', 'repo_name': 'wesleykaoru/npl_nltk', 'size': '5654'}