<jupyter_start><jupyter_text>1. Importacao das bibliotecas<jupyter_code>import joblib
import matplotlib.pyplot as plt
import pandas as pd
import seaborn as sns

from sklearn.feature_extraction.text import TfidfVectorizer
from sklearn.metrics import accuracy_score, classification_report
from sklearn.model_selection import train_test_split
from sklearn.preprocessing import LabelEncoder
from sklearn.pipeline import Pipeline
from sklearn.svm import LinearSVC<jupyter_output><empty_output><jupyter_text>2. Importando os dados<jupyter_code>headlines_df = pd.read_csv('/content/drive/MyDrive/NLP/dados/news_headlines.csv')
headlines_df.reset_index(drop=True, inplace=True) #Apaga indices caso existam<jupyter_output><empty_output><jupyter_text>3. Visualizacao dos dados<jupyter_code>headlines_df.info()
headlines_df.head()
headlines_df.tail()
sns.countplot(y='tags', data=headlines_df, palette='pastel')
plt.show()<jupyter_output><empty_output><jupyter_text>4. Convertendo rotulos<jupyter_code>le = LabelEncoder()
Y = le.fit_transform(headlines_df['tags'])
le.classes_<jupyter_output><empty_output><jupyter_text>5. Separando dados de treino e teste<jupyter_code>x_train, x_test, y_train, y_test = train_test_split(headlines_df['post'], Y, test_size=0.2, random_state=5)<jupyter_output><empty_output><jupyter_text>6. Definindo o Pipeline<jupyter_code>pipeline = Pipeline([
                     ('vectorizer', TfidfVectorizer()), # Features
                     ('clf', LinearSVC()), # Classificador
])
pipeline.steps<jupyter_output><empty_output><jupyter_text>7. Treinamento do modelo<jupyter_code>pipeline.fit(x_test, y_test)<jupyter_output><empty_output><jupyter_text>8. Avaliacao do Modelo<jupyter_code>pred = pipeline.predict(x_test)
pred
accuracy_score(y_test, pred) # Acuracia geral
print(classification_report(y_test, pred)) # Acuracia especifica por classe
le.classes_<jupyter_output><empty_output><jupyter_text>9. Predicao<jupyter_code>text = 'Sao Paulo Futebol Clube foi campeao paulista de 2021.'

prediction = pipeline.predict([text])
le.inverse_transform(prediction)[0]
text = 'Doria desiste de candidatura a Presidencia em 2022.'
prediction = pipeline.predict([text])
le.inverse_transform(prediction)[0]<jupyter_output><empty_output><jupyter_text>10. Exportando modelo para uso em producao<jupyter_code>joblib.dump(pipeline, "model.joblib")<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.10630252100840336, 'fraction_numerical': 0.010084033613445379, 'mean_word_length': 8.562248995983936, 'pattern_counts': {'":': 0, '<': 41, '<?xml version=': 0, '>': 41, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '10', 'dir': 'jupyter-structured-clean-dedup', 'id': '576974', 'license': 'MIT', 'n_tokens_mistral': 874, 'n_tokens_neox': 858, 'n_words': 143, 'path': 'classificador_de_textos.ipynb', 'repo_name': 'joscelino/Portuguese_NLP', 'size': '2380'}