<jupyter_start><jupyter_text>NEWS CLASSIFICATION<jupyter_code>import pandas as pd
import seaborn as sns
from sklearn.model_selection import train_test_split
import numpy as np
from sklearn.feature_extraction.text import TfidfVectorizer
from sklearn import svm
from sklearn.feature_extraction.text import TfidfTransformer
from sklearn.pipeline import Pipeline<jupyter_output><empty_output><jupyter_text>The Dataset : NewsDataset.json news=2,00,853<jupyter_code>allNews = pd.read_json('NewsDataset.json',lines=True)
allNews.info()
allNews.head()
allNews.describe()<jupyter_output><empty_output><jupyter_text>Dropping The Unwanted Features from the dataset<jupyter_code>droplist=['authors','date','link']
for todrop in droplist:
    allNews = allNews.drop(todrop,1)
allNews.info()
allNews.head()
allNews.columns
allNews.columns=['CATEGORY','TITLE','DESC']
allNews.head()
allNews.info()
labels_of_news=[]
x = allNews['CATEGORY'].value_counts()
for i in x.index:
    labels_of_news.append(i)
allNews['CATEGORY'].value_counts()
print("value_counts",len(allNews['CATEGORY'].value_counts()))
print("label_of_news",len(labels_of_news))
labels_of_news<jupyter_output><empty_output><jupyter_text>- Checking the category and their title and description<jupyter_code>wellness = allNews.loc[allNews['CATEGORY'] == "WELLNESS"]
wellness.head() #Remove it 
parent =  allNews.loc[allNews['CATEGORY'] == "PARENTS"]
parent.head() # remove it 

env =  allNews.loc[allNews['CATEGORY'] == "ENVIRONMENT"]
env.head() # merge with ENTERTAINMENT
divorce =  allNews.loc[allNews['CATEGORY'] == "DIVORCE"]
divorce.head() # remove it 
taste =  allNews.loc[allNews['CATEGORY'] == "TASTE"]
taste.head() # remove it
impact =  allNews.loc[allNews['CATEGORY'] == "IMPACT"]
impact.head() # remove it
# WEDDINGS
wed =  allNews.loc[allNews['CATEGORY'] == "WEDDINGS"]
wed.head() # remove it
## Merging Categories

#worldpost and 'the worldpost'
allNews.CATEGORY = allNews.CATEGORY.map(lambda x: "WORLDPOST" if x == "THE WORLDPOST" else x)
allNews.CATEGORY = allNews.CATEGORY.map(lambda x: "ENTERTAINMENT" if x == "COMEDY" else x)
allNews.CATEGORY = allNews.CATEGORY.map(lambda x: "ARTS_CULTURE" if x == "ARTS" or x=="CULTURE & ARTS" or x=="ARTS & CULTURE" else x)

allNews.CATEGORY = allNews.CATEGORY.map(lambda x: "WORLDNEWS" if x == "WORLDPOST" or x == "WORLD NEWS"  else x)

len(allNews.CATEGORY.value_counts())
(allNews.CATEGORY.value_counts())<jupyter_output><empty_output><jupyter_text>-  NEEDED LABELS ONLY<jupyter_code>final_category = ["POLITICS","ENTERTAINMENT","WORLDNEWS","SPORTS","BUSINESS","TECH","SCIENCE","ARTS_CULTURE"]
len(final_category)
myNewsDataset =  allNews.loc[allNews['CATEGORY'].isin(final_category)]
myNewsDataset.head()
myNewsDataset.info()

myNewsDataset.describe()
myNewsDataset.CATEGORY.value_counts()
myNewsDataset["data_unparsed"]=myNewsDataset["TITLE"] + " " + myNewsDataset["DESC"]
droplist=['TITLE','DESC']
for todrop in droplist:
    myNewsDataset = myNewsDataset.drop(todrop,1)
myNewsDataset['data_unparsed']=myNewsDataset.data_unparsed.str.lower()
## final news count = 81,351

myNewsDataset.info()

myNewsDataset.describe()

myNewsDataset.head()
remove_this=("“","’s","s’","’t")
punctuation_sign = ("!","?","@","?",".",":",";","-","&",",","%") 
myNewsDataset['data']=myNewsDataset['data_unparsed'].str.replace("\n"," ")

myNewsDataset['data']=myNewsDataset['data'].str.replace("\r"," ")

myNewsDataset['data']=myNewsDataset['data'].str.replace('"','')

for remove_ in remove_this:    
    myNewsDataset['data']=myNewsDataset['data'].str.replace(remove_,"")
for punc_ in punctuation_sign:
    myNewsDataset['data']=myNewsDataset['data'].str.replace(punc_,"")
myNewsDataset['data']=myNewsDataset['data'].str.replace("'t","")
myNewsDataset['data']=myNewsDataset['data'].str.replace("'s","")
myNewsDataset['data']=myNewsDataset['data'].str.replace("'","")

myNewsDataset.head()
myNewsDataset = myNewsDataset.drop("data_unparsed",1)

text=myNewsDataset.loc[27205]['data']
# print(text)
# text.replace("\'","")
text
myNewsDataset.describe()

myNewsDataset.drop_duplicates(keep=False, inplace=True)
myNewsDataset.describe()

myNewsDataset=myNewsDataset[myNewsDataset.data != " "]
myNewsDataset.describe()

myNewsDataset.head()
myNewsDataset['Category_label']  =myNewsDataset.CATEGORY.map({'POLITICS':0,
                            'ENTERTAINMENT':1,
                            'WORLDNEWS':2,         
                            'BUSINESS':3,          
                            'SPORTS':4,            
                            'ARTS_CULTURE':5,      
                            'SCIENCE':6,           
                            'TECH':7})
myNewsDataset.columns
n_array=myNewsDataset.as_matrix()
x_data =n_array[:,1]
y_label =n_array[:,2]
y_label = y_label.astype('int')
myNewsDataset.head()<jupyter_output><empty_output><jupyter_text>----------------- Cleaning Ends ---------------<jupyter_code># visualizing
sns.set_style("whitegrid")

sns.countplot(y=myNewsDataset.CATEGORY,linewidth=5,)
len(myNewsDataset.CATEGORY)

value_counts=myNewsDataset.CATEGORY.value_counts()
# int(value_counts)
for v in value_counts:
    print(int(v))<jupyter_output>32700
21209
8417
5932
4884
3874
2178
2028<jupyter_text>Spliting The Dataset<jupyter_code>train_set,test_set = train_test_split(myNewsDataset,test_size=0.2,random_state=42)
train_set.CATEGORY.value_counts()
# test_set.CATEGORY.value_counts()
from sklearn.model_selection import StratifiedShuffleSplit

split = StratifiedShuffleSplit(n_splits=1, test_size=0.2, random_state=42)
for train_index, test_index in split.split(myNewsDataset, myNewsDataset["Category_label"]):
    strat_train_set = myNewsDataset.iloc[train_index]
    strat_test_set = myNewsDataset.iloc[test_index]
strat_train_set.head()
strat_test_set.info()
# strat_train_set == is our training set
# strat_test_set == test set<jupyter_output><empty_output><jupyter_text>CountVectorizer<jupyter_code>from sklearn.feature_extraction.text import CountVectorizer

cvec = CountVectorizer(stop_words="english")
x_train_data = cvec.fit(strat_train_set.data)
x_train_transform = cvec.transform(strat_train_set.data)
#transform the x_test data as per CountVectorizer from x_train data
x_test_transform = cvec.transform(strat_test_set.data)
x_train_transform
x_test_transform
y_train = strat_train_set.Category_label
y_test = strat_test_set.Category_label<jupyter_output><empty_output><jupyter_text>Multinomial NB<jupyter_code>from sklearn.naive_bayes import MultinomialNB
from sklearn.model_selection import GridSearchCV
from sklearn.metrics import classification_report, confusion_matrix, accuracy_score
NBmodel = MultinomialNB()

# fitting
NBmodel.fit(x_train_transform,y_train)
#Predicting x_test set
y_pred_class = NBmodel.predict(x_test_transform)
length=0
for i,j in zip(y_pred_class,y_test):
    length+=1
    print(i," -------- ",j)
    if length==5:
        break
# measuring accuracy
from sklearn import metrics
metrics.accuracy_score(y_test,y_pred_class)
# confusion matrix
confusion_matrix = metrics.confusion_matrix(y_test,y_pred_class)
print(confusion_matrix)
confusion_matrix.shape
# Classification report
print("Classification report")
print(classification_report(y_test,y_pred_class))<jupyter_output>Classification report
              precision    recall  f1-score   support

           0       0.82      0.90      0.86      6540
           1       0.78      0.88      0.83      4242
           2       0.76      0.74      0.75      1683
           3       0.68      0.63      0.66      1186
           4       0.92      0.68      0.78       977
           5       0.83      0.46      0.59       775
           6       0.91      0.49      0.64       436
           7       0.80      0.29      0.42       406

   micro avg       0.80      0.80      0.80     16245
   macro avg       0.81      0.63      0.69     16245
weighted avg       0.80      0.80      0.79     16245<jupyter_text>Measurement- Accuracy: the accuracy metric measures the ratio of correct predictions over the total number of instances evaluated.- Precision: precision is used to measure the positive patterns that are correctly predicted from the total predicted patterns in a positive class.- Recall: recall is used to measure the fraction of positive patterns that are correctly classified- F1-Score: this metric represents the harmonic mean between recall and precision values<jupyter_code># 'POLITICS':0,
# 'ENTERTAINMENT':1,
# 'WORLDNEWS':2,         
# 'BUSINESS':3,          
# 'SPORTS':4,            
# 'ARTS_CULTURE':5,      
# 'SCIENCE':6,           
# 'TECH':7
import seaborn as sns;
heat_map = sns.heatmap(confusion_matrix,cmap="YlGnBu",  annot=True,

                       xticklabels=["POLITICS","ENTERTAINMENT","WORLDNEWS","BUSINESS","SPORTS","ArtsC","Science","TECH"],
                       yticklabels=["POLITICS","ENTERTAINMENT","WORLDNEWS","BUSINESS","SPORTS","ArtsC","Science","TECH"])
n_m = confusion_matrix / confusion_matrix.astype(np.float).sum(axis=1)
clean_n_m=(np.around(n_m,decimals=3))
clean_n_m
import seaborn as sns;

heat_map = sns.heatmap(clean_n_m,cmap="YlGnBu",
                       xticklabels=["POLITICS","ENTERTAINMENT","WORLDNEWS","BUSINESS","SPORTS","ArtsC","Science","TECH"],
                       yticklabels=["POLITICS","ENTERTAINMENT","WORLDNEWS","BUSINESS","SPORTS","ArtsC","Science","TECH"])<jupyter_output><empty_output><jupyter_text>My test<jupyter_code>doc= ["Patriots' Cardona promoted to lieutenant in Navy Joe Cardona, who has spent his entire NFL career with the Patriots balancing his football and active-duty commitments, has been promoted to the rank of lieutenant in the U.S. Navy."]
docx = cvec.transform(doc)
print(docx)
docp = NBmodel.predict(docx)
docp
# Pipeplines 
myNBmodel = Pipeline([('cvec',CountVectorizer(stop_words="english")),
                       ('clf',MultinomialNB()),
                     ])
myNBmodel.fit(strat_train_set.data,y_train)
y_pred_class = myNBmodel.predict(strat_test_set.data)
# measuring accuracy
from sklearn import metrics
metrics.accuracy_score(y_test,y_pred_class)
p =myNBmodel.predict(doc)
import pickle
# Saving model to disk
pickle.dump(myNBmodel, open('myNBmodel.pkl','wb'))


# Loading model to compare the results
myNBmodel = pickle.load( open('myNBmodel.pkl','rb'))
myNBmodel.predict(doc)<jupyter_output><empty_output><jupyter_text>Complement NB - Now, for Complement Normal Naive Bayes, instead of calculating the likelihood of a word occuring in a class, we calculate the likelihood that it occurs in other classes. So, we would proceed to calculate the word-class dependencies- So, the one with the lower probability (minimum value) is said to be the class it belongs to . We DON'T use the one with the maximum probability because for the Complement Naive Bayes Algorithm, we take it - a higher value - to mean that it is highly likely that a document with these words does NOT belong to that class.<jupyter_code>from sklearn.naive_bayes import ComplementNB
pcomp = Pipeline([('tfidf', TfidfVectorizer(stop_words="english")),
               ('cnb',ComplementNB()),
              ])

pcomp.fit(strat_train_set.data,y_train)

y_pred_class=pcomp.predict(strat_test_set.data)
# measuring accuracy
from sklearn import metrics
metrics.accuracy_score(y_test,y_pred_class)
# confusion matrix
confusion_matrix = metrics.confusion_matrix(y_test,y_pred_class)
print(confusion_matrix)<jupyter_output>[[5951  219  235   68   34   17    5   11]
 [ 339 3767   24   34   27   29    9   13]
 [ 355   58 1194   23   15   18   15    5]
 [ 321  102   46  664   16   16    2   19]
 [  77  120   25    8  745    2    0    0]
 [  43  280   27   19    6  385   10    5]
 [  52   50   14   11    6    7  291    5]
 [ 101   62   13   59    1    7    7  156]]<jupyter_text>----count<jupyter_code># Pipeplines 
myCBmodel = Pipeline([('cvec',CountVectorizer(stop_words="english")),
                       ('clf',ComplementNB()),
                     ])
myCBmodel.fit(strat_train_set.data,y_train)

y_pred_class=myCBmodel.predict(strat_test_set.data)
# measuring accuracy
from sklearn import metrics
metrics.accuracy_score(y_test,y_pred_class)
confusion_matrix = metrics.confusion_matrix(y_test,y_pred_class)
print(confusion_matrix)
import seaborn as sns;
heat_map = sns.heatmap(confusion_matrix,color="Blue",annot=True,
                       xticklabels=["POLITICS","ENTERTAINMENT","WORLDNEWS","BUSINESS","SPORTS","ArtsC","Science","TECH"],
                       yticklabels=["POLITICS","ENTERTAINMENT","WORLDNEWS","BUSINESS","SPORTS","ArtsC","Science","TECH"])
# Classification report
print("Classification report")
print(classification_report(y_test,y_pred_class))

import pickle

pickle.dump(myCBmodel, open('myCBmodel.pkl','wb'))<jupyter_output><empty_output><jupyter_text>---  GOOD BYE ---  Support Vector Machine<jupyter_code>from sklearn.svm import LinearSVC
sv = Pipeline([('Tfidf',TfidfVectorizer()),('svm',LinearSVC())])
sv
sv.fit(strat_train_set.data,y_train)

y_pred_class=sv.predict(strat_test_set.data)
# measuring accuracy
from sklearn import metrics
metrics.accuracy_score(y_test,y_pred_class)
# confusion matrix
confusion_matrix = metrics.confusion_matrix(y_test,y_pred_class)
print(confusion_matrix)
# Classification report
print("Classification report")
print(classification_report(y_test,y_pred_class))<jupyter_output>Classification report
              precision    recall  f1-score   support

           0       0.85      0.90      0.87      6540
           1       0.82      0.87      0.85      4242
           2       0.76      0.74      0.75      1683
           3       0.70      0.65      0.68      1186
           4       0.86      0.78      0.82       977
           5       0.80      0.58      0.67       775
           6       0.83      0.67      0.74       436
           7       0.69      0.52      0.59       406

   micro avg       0.82      0.82      0.82     16245
   macro avg       0.79      0.71      0.75     16245
weighted avg       0.82      0.82      0.81     16245<jupyter_text>---Logistic Regeresssion---<jupyter_code>from sklearn.linear_model import LogisticRegression

lr= LogisticRegression(C=1.0, class_weight='balanced', dual=False,
          fit_intercept=True, intercept_scaling=1, max_iter=100,
          multi_class='multinomial', n_jobs=1, penalty='l2',
          random_state=8, solver='sag', tol=0.0001, verbose=0,
          warm_start=False)



lr.fit(x_train_transform,y_train)


y_pred_class = lr.predict(x_test_transform)


# measuring accuracy
from sklearn import metrics
metrics.accuracy_score(y_test,y_pred_class)

sv = Pipeline([('tfidf', TfidfVectorizer()),
               ('clf',LogisticRegression(C=1.0, class_weight='balanced',
                dual=False,fit_intercept=True, intercept_scaling=1,
                max_iter=100,multi_class='multinomial', n_jobs=1,
                penalty='l2',random_state=8, solver='sag', tol=0.0001, verbose=0,warm_start=False))
              ])
               
sv.fit(strat_train_set.data,y_train)

y_pred_class=sv.predict(strat_test_set.data)
# measuring accuracy
from sklearn import metrics
metrics.accuracy_score(y_test,y_pred_class)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.12256488146338242, 'fraction_numerical': 0.03638644918444166, 'mean_word_length': 3.9248780487804877, 'pattern_counts': {'":': 1, '<': 57, '<?xml version=': 0, '>': 57, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 16, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '15', 'dir': 'jupyter-structured-clean-dedup', 'id': '95980', 'license': 'MIT', 'n_tokens_mistral': 5580, 'n_tokens_neox': 5257, 'n_words': 982, 'path': 'NewsClassification.ipynb', 'repo_name': 'utsavdarlami/BreakfastScoop', 'size': '15143'}