<jupyter_start><jupyter_text>Classification new using Naive-Bayes<jupyter_code>import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
import warnings
warnings.filterwarnings('ignore')
%matplotlib inline
%config InlineBackend.figure_format = 'retina'

np.random.seed(123456)
!ls<jupyter_output>comment_spam.csv    doc.pdf	 'News Title.xls'   'Task Data Scientist.doc'
'Comment Spam.xls'   news.csv	  spam-Copy1.ipynb
 doc.doc	     news.ipynb   spam.ipynb<jupyter_text>Read Dataset<jupyter_code>news = pd.read_csv('news.csv')
news.tail()
news['Category'].unique()
news['Category'].value_counts().plot(kind='bar')
plt.show()<jupyter_output><empty_output><jupyter_text>Data Normalize<jupyter_code>import re
import string

def clean_text(s):
    s = s.lower()
    for ch in string.punctuation:                                                                                                     
        s = s.replace(ch, " ") 
    s = re.sub("[0-9]+", "||DIG||",s)
    s = re.sub(' +',' ', s)        
    return s

news['CLEAN'] = [clean_text(s) for s in news['News Title']]
#news['TEXT']
from sklearn.feature_extraction.text import CountVectorizer
from sklearn.feature_extraction.text import TfidfVectorizer, TfidfTransformer
from sklearn.preprocessing import LabelEncoder
from sklearn.naive_bayes import MultinomialNB
from sklearn.model_selection import train_test_split
from sklearn.metrics import confusion_matrix
from sklearn.metrics import classification_report
from sklearn.model_selection import cross_val_score
from sklearn.tree import DecisionTreeClassifier
# pull the data into vectors
vectorizer = CountVectorizer()
x = vectorizer.fit_transform(news['CLEAN'])

# for Tfidf (we have tried and the results aren't better)
#tfidf = TfidfVectorizer()
#x = tfidf.fit_transform(dataset['TEXT'].values)

encoder = LabelEncoder()
y = encoder.fit_transform(news['Category'])

# split into train and test sets
x_train, x_test, y_train, y_test = train_test_split(x, y, test_size=0.2)

# take a look at the shape of each of these
print(x_train.shape)
print(y_train.shape)
print(x_test.shape)
print(y_test.shape)<jupyter_output>(52428, 26383)
(52428,)
(13107, 26383)
(13107,)<jupyter_text>Train and Check Naive bayes model<jupyter_code>%%time
nb = MultinomialNB()
nb.fit(x_train, y_train)
results_nb_cv = cross_val_score(nb, x_train, y_train, cv=10)
test_acc = nb.score(x_test, y_test)
print('Cross Validation: ', results_nb_cv.mean())
print('Test data Acc: ',test_acc)
x_test_pred = MultinomialNB().fit(x_train, y_train).predict(x_test)
confusion_matrix(y_test, x_test_pred)
print(classification_report(y_test, x_test_pred, target_names=encoder.classes_))<jupyter_output>precision    recall  f1-score   support

     Business       0.89      0.89      0.89      3580
Entertainment       0.95      0.95      0.95      4716
      Medical       0.90      0.86      0.88      1384
   Technology       0.88      0.89      0.89      3427

    micro avg       0.91      0.91      0.91     13107
    macro avg       0.91      0.90      0.90     13107
 weighted avg       0.91      0.91      0.91     13107<jupyter_text>Try to predict news<jupyter_code>def predict_cat(title):
    #cat_names = {'b' : 'business', 't' : 'science and technology', 'e' : 'entertainment', 'm' : 'health'}
    cat_names = {'Technology' : 'Technology News', 'Business' : 'Business News', 'Entertainment' : 'Entertainment News', 'Medical' : 'Medical News'}
    cod = MultinomialNB().fit(x_train, y_train).predict(vectorizer.transform([title]))
    return cat_names[encoder.inverse_transform(cod)[0]]
news['Category'].unique()
predict_cat('<NAME> 3 has failed to get attentions')
predict_cat('The drama surrounding Google and Huawei\'s new phone – explained')
predict_cat('China starts to impose additional tariffs on some U.S. goods')<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.11423367161072079, 'fraction_numerical': 0.03642987249544627, 'mean_word_length': 3.9030612244897958, 'pattern_counts': {'":': 0, '<': 19, '<?xml version=': 0, '>': 19, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '5', 'dir': 'jupyter-structured-clean-dedup', 'id': '210260', 'license': 'MIT', 'n_tokens_mistral': 1399, 'n_tokens_neox': 1300, 'n_words': 304, 'path': 'news.ipynb', 'repo_name': 'shuistama/headline-news-classification', 'size': '3846'}