<jupyter_start><jupyter_text>Análise de Sentimento baseado no dataset pego no Twitter  Importando as libraries<jupyter_code>import numpy as np
import pandas as pd
from tqdm import tqdm<jupyter_output><empty_output><jupyter_text>Puxando o dataset<jupyter_code>dataset = pd.read_csv("tweetSentimentBR.txt", sep='\t', header=None)
dataset.columns = ['Tweet ID', 'Hashtag', 'Vetor', '?', 'Sentimento']
dataset = dataset.drop(['Vetor', '?'], axis=1)
msgs = pd.read_csv("output.txt", '\t', header=None)
msgs = np.array(msgs)
dataset.head()
msgs[:5]
print(msgs.shape)
print(dataset.shape)
dataset_np = np.array(dataset)
dataset_np<jupyter_output><empty_output><jupyter_text>Tratamento dos dados<jupyter_code>import math
msgs_new = []

for dado in msgs:
    if dado[1] == str('nan') or dado[0] == '?':
        continue
    else:
        msgs_new.append(dado)

msgs_new = np.array(msgs_new)
msgs_new.shape
counter_nan = 0 
counter_inter = 0

for dado in msgs_new:
    if dado[0] == str('nan'):
        counter_nan += 1
    if dado[1] == '?':
        counter_inter += 1
        
print(counter_inter)
print(counter_nan)<jupyter_output>0
0<jupyter_text>Colocando os dois datasets juntos<jupyter_code>data_final = []

for dado_dataset in tqdm(dataset_np):
    for dado_msgs in msgs_new:
        if str(dado_dataset[0]) == dado_msgs[0]:
            aux = [dado_dataset[0], dado_dataset[1], dado_dataset[2], dado_msgs[1]]
            data_final.append(aux)
data_final = np.array(data_final)
data_final.shape
data_final[:5]
df = pd.DataFrame(data_final)
df.columns = ['Tweet ID', 'Hashtag', 'Sentiment', 'Texto']
df
df = df.drop(['Tweet ID', 'Hashtag'], axis=1)
df
df.to_csv('dataFinal.csv', sep=',', header=['Sentimento', 'Texto'], index=False)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.15008527572484365, 'fraction_numerical': 0.01137009664582149, 'mean_word_length': 5.153846153846154, 'pattern_counts': {'":': 0, '<': 16, '<?xml version=': 0, '>': 16, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '4', 'dir': 'jupyter-structured-clean-dedup', 'id': '379020', 'license': 'Apache-2.0', 'n_tokens_mistral': 701, 'n_tokens_neox': 692, 'n_words': 134, 'path': 'Tratamento da Base de Dados.ipynb', 'repo_name': 'MarcelloRCestaro/sentiment_analysis_twitter', 'size': '1759'}