<jupyter_start><jupyter_text>Predict Category of News Using Labeled NYT Data- Take out the rows that are in category :-  'Gen Soft', 'Misc', 'Week in Review', 'Magazine', 'Home Desk', 'Cars', 'Living', 'Personal Finance'- Take out the rows that are shorter than 150 characters- Stratified random sampling 7000 rows for each categories (17 categories)- Split 80% for trained dataset and 20% for test dataset- Feature extraction (unigram/bigram/trigram) by take out the n-grams that are occurrence less than 0.05% or greater than 50% of trained dataset- Fit the model using LinearSVC<jupyter_code>import pandas as pd
df = pd.read_csv('../../../nyt_data/nyt_recode_clean.csv.bz2', nrows=10)
select_cols = "categories, OnlineMSoft, NewsDeskSoft, Publication.Date, Publication.Year, Section, Body, Lead.Paragraph, Headline, Online.Headline, Online.Lead.Paragraph, Url, Online.Section, ID"
select_cols = [c.strip() for c in select_cols.split(',')]
for c in select_cols:
    if c not in df.columns:
        print(c)
df = pd.read_csv('../../../nyt_data/nyt_recode_clean.csv.bz2', usecols=select_cols)
df.dropna(subset=['Body'], inplace=True)
df
df.groupby('categories').size()<jupyter_output><empty_output><jupyter_text>Let all other categories ==> Other<jupyter_code>df.loc[df.categories.isnull() | df.categories.isin(['Gen Soft', 'Misc', 'Week in Review', 'Magazine', 'Home Desk', 'Cars', 'Living', 'Personal Finance']), 'categories'] = 'Other'
df.groupby('categories').size()
pd.set_option('max_colwidth', 120)
df[df.Body.str.len() < 150][['Body', 'Url']]<jupyter_output><empty_output><jupyter_text>Take out the Body text shorter than 150 characters<jupyter_code>df = df[df.Body.str.len() > 150]
df.groupby(['categories']).size()
!pip install nltk
import time
import re
import string

import numpy as np
import nltk
from nltk import word_tokenize          
from nltk.stem.porter import PorterStemmer

from sklearn.feature_extraction.text import CountVectorizer
from sklearn.feature_extraction.text import TfidfTransformer
from sklearn.metrics import classification_report, accuracy_score, f1_score
from sklearn.model_selection import train_test_split
from sklearn.externals import joblib

stemmer = PorterStemmer()
def stem_tokens(tokens, stemmer):
    stemmed = []
    for item in tokens:
        stemmed.append(stemmer.stem(item))
    return stemmed

def tokenize_with_punc(text):
    tokens = nltk.word_tokenize(text)
    stems = stem_tokens(tokens, stemmer)
    return stems

def tokenize(text):
    text = "".join([ch for ch in text if ch not in string.punctuation])
    tokens = nltk.word_tokenize(text)
    stems = stem_tokens(tokens, stemmer)
    return stems

with open('../../../roberts_rules/all_text.txt', 'rt') as f:
    text = f.read()
text = re.sub(r'\d+', '', text)

vect = CountVectorizer(tokenizer=tokenize, stop_words='english', ngram_range=(2, 3)) 
vect.fit([text])
roberts_rules = set(vect.get_feature_names())


def most_informative_feature_for_class(vectorizer, classifier, classlabel, n=10):
    labelid = list(classifier.classes_).index(classlabel)
    feature_names = vectorizer.get_feature_names()
    topn = sorted(zip(classifier.coef_[labelid], feature_names))[-n:]

    for coef, feat in topn:
        print(classlabel, feat, coef)

def most_informative_feature_for_class_svm(vectorizer, classifier,  n=10):
    labelid = 3 # this is the coef we're interested in. 
    feature_names = vectorizer.get_feature_names()
    svm_coef = classifier.coef_.toarray() 
    topn = sorted(zip(svm_coef[labelid], feature_names))[-n:]

    for coef, feat in topn:
        print(feat, coef)
        
def print_top10(vectorizer, clf, class_labels):
    """Prints features with the highest coefficient values, per class"""
    feature_names = vectorizer.get_feature_names()
    for i, class_label in enumerate(class_labels):
        top10 = np.argsort(clf.coef_[i])[-10:]
        print("%s: %s" % (class_label,
              " | ".join(feature_names[j] for j in top10)))

def get_top_features(vectorizer, clf, class_labels, n=20):
    """Prints features with the highest coefficient values, per class"""
    feature_names = vectorizer.get_feature_names()
    top_features = {}
    for i, class_label in enumerate(class_labels):
        topN = np.argsort(clf.coef_[i])[-n:]
        top_features[class_label] = [feature_names[j] for j in topN][::-1]
    return top_features

def show_most_informative_features(vectorizer, clf, n=20):
    feature_names = vectorizer.get_feature_names()
    coefs_with_fns = sorted(zip(clf.coef_[0], feature_names))
    top = zip(coefs_with_fns[:n], coefs_with_fns[:-(n + 1):-1])
    for (coef_1, fn_1), (coef_2, fn_2) in top:
        print("\t%.4f\t%-20s\t\t%.4f\t%-20s" % (coef_1, fn_1, coef_2, fn_2))

def get_most_informative_features(vectorizer, clf, n=20):
    feature_names = vectorizer.get_feature_names()
    coefs_with_fns = sorted(zip(clf.coef_[0], feature_names))
    top_a = coefs_with_fns[:n]
    top_b = coefs_with_fns[:-(n + 1):-1]
    return top_a, top_b<jupyter_output>/opt/venv37/lib/python3.7/site-packages/sklearn/externals/joblib/__init__.py:15: FutureWarning: sklearn.externals.joblib is deprecated in 0.21 and will be removed in 0.23. Please import this functionality directly from joblib, which can be installed with: pip install joblib. If this warning is raised when loading pickled models, you may need to re-serialize those models with scikit-learn 0.21+.
  warnings.warn(msg, category=FutureWarning)
/opt/venv37/lib/python3.7/site-packages/sklearn/feature_extraction/text.py:507: UserWarning: The parameter 'token_pattern' will not be used since 'tokenizer' is not None'
  warnings.warn("The parameter 'token_pattern' will not be used"
/opt/venv37/lib/python3.7/site-packages/sklearn/feature_extraction/text.py:385: UserWarning: Your stop_words may be inconsistent with your preprocessing. Tokenizing the stop words generated tokens ['abov', 'afterward', 'alon', 'alreadi', 'alway', 'ani', 'anoth', 'anyon', 'anyth', 'anywher', 'becam', 'becaus', 'becom', '[...]<jupyter_text>Stratified Sampling<jupyter_code>#SAMPLE_SIZE_PER_CAT = 4000 ==> 0.85
#SAMPLE_SIZE_PER_CAT = 6000 ==> 0.86
SAMPLE_SIZE_PER_CAT = 7000
sdf = pd.DataFrame()
for c in df.categories.unique():
    sdf = sdf.append(df[df.categories == c].sample(SAMPLE_SIZE_PER_CAT, random_state=21, replace=True))
sdf.groupby(['categories']).size()
sdf[['ID', 'Body', 'categories']].to_csv('nyt_sample_7k.csv', index=False)<jupyter_output><empty_output><jupyter_text>X = sdf.Bodyy = sdf.categoriesX_train,  X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=21, stratify=y)<jupyter_code>X = sdf[['ID', 'Body']]
y = sdf.categories

X_train,  X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=21, stratify=y)
X_test_data = X_test.copy()
X_train = X_train.Body
X_test = X_test.Body
import re

def custom_tokenizer(doc):
    doc = re.sub('\d+', '[NUM]', doc)
    return doc.split()
vect = CountVectorizer(ngram_range=(1, 3), min_df=0.0005, max_df=0.5, tokenizer=custom_tokenizer, max_features=20000)
#vect = CountVectorizer(ngram_range=(1, 3), min_df=10, max_df=0.3, tokenizer=custom_tokenizer, max_features=10000)
#vect = CountVectorizer(ngram_range=(1, 3), min_df=200, max_df=0.3, tokenizer=custom_tokenizer)
%%time
X_train = vect.fit_transform(X_train)
transformer = TfidfTransformer()
X_train = transformer.fit_transform(X_train)
len(vect.vocabulary_)
%%time
X_test = vect.transform(X_test)
transformer = TfidfTransformer()
X_test = transformer.fit_transform(X_test)
len(vect.vocabulary_)
%%time
from sklearn.svm import LinearSVC
from sklearn.calibration import CalibratedClassifierCV

est = LinearSVC(penalty='l1', dual=False, tol=1e-3)

# Calibrated with isotonic calibration
clf = CalibratedClassifierCV(est, cv=2, method='isotonic')

t0 = time.time()
clf.fit(X_train, y_train)
t1 = time.time()
y_pred = clf.predict(X_test)
t2 = time.time()
time_clf_train = t1-t0
time_clf_predict = t2-t1
print("Results for classifier")
print("Training time: %fs; Prediction time: %fs" % (time_clf_train, time_clf_predict))
print(classification_report(y_test, y_pred))
f1_score(y_test, y_pred, average='macro')
len(vect.vocabulary_)
# FIXME: Only for first fold of cross validation
most_informative_feature_for_class(vect, clf.calibrated_classifiers_[0].base_estimator, 'Arts')
print_top10(vect, clf.calibrated_classifiers_[0].base_estimator, clf.classes_)
top_features = get_top_features(vect, clf.calibrated_classifiers_[0].base_estimator, clf.classes_, n=100)
top_features_df = pd.DataFrame(top_features)
top_features_df.to_csv('./reports/us_categories_top100_other_calibrated.csv', index=False)
top_features_df
%%time
vect.stop_words_ = None
joblib.dump(vect, "../data/us_model/nyt_us_vectorizer.joblib", compress=3)
joblib.dump(clf, "../data/us_model/nyt_us_classifier.joblib", compress=3)
y_test_df = pd.DataFrame(y_test)
y_test_df.columns = ['true_value']
y_test_df.reset_index(drop=True, inplace=True)

y_test_df['pred_value'] = clf.predict(X_test)

if hasattr(clf, "predict_proba"):
    prob = clf.predict_proba(X_test)
else:  # use decision function
    prob = clf.decision_function(X_test)
    prob = \
        (prob - prob.min()) / (prob.max() - prob.min())
prob_df = pd.DataFrame(prob)
columns = []
for c in clf.classes_:
    columns.append(c)
prob_df.columns = columns
result_df = pd.concat([X_test_data.reset_index(drop=True), y_test_df, prob_df], axis=1)
result_df.to_csv('./tests/us_categories_test_prediction_other_calibrated+text.csv', index=False)
result_df
result_df[result_df.pred_value!=result_df.true_value].to_csv('./reports/nyt_test_pred_misclass.csv', index=False)
%matplotlib inline
import matplotlib.pyplot as plt
from pandas_confusion import ConfusionMatrix

y_true = y_test.reset_index(drop=True)
confusion_matrix = ConfusionMatrix(y_true, y_pred)
print("Confusion matrix:\n%s" % confusion_matrix)
confusion_matrix.plot()
from sklearn.metrics import confusion_matrix
conf = confusion_matrix(y_true, y_pred)
conf_df = pd.DataFrame(conf)
conf_df.columns = clf.classes_
conf_df.index = clf.classes_
conf_df.to_csv('./reports/us_categories_test_confusion_matrix_other_calibrated.csv', index_label="actual \ predicted")
print(conf_df)<jupyter_output>Arts  Books  Business Finance  Classifieds  Dining  \
Arts               948     37                 9            6       2   
Books               11   1330                 3            0       3   
Business Finance    10      0              1202            0       5   
Classifieds          1      0                 0         1391       0   
Dining               4      3                 0            0    1346   
Editorial           16     11                15            0       3   
Foreign News        13      2                17            0       2   
Health               1      1                 3            0       5   
Leisure            265     12                 1            0      23   
Local               56      6                24            2      41   
National            13      4                41            2       3   
Obits               11      2                 1            1       1   
Other               78     33                38            5    [...]
{'fraction_non_alphanumeric': 0.11947612259444049, 'fraction_numerical': 0.02601568068424804, 'mean_word_length': 3.9210872424375274, 'pattern_counts': {'":': 0, '<': 20, '<?xml version=': 0, '>': 23, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 25, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '5', 'dir': 'jupyter-structured-clean-dedup', 'id': '103950', 'license': 'MIT', 'n_tokens_mistral': 4094, 'n_tokens_neox': 3864, 'n_words': 868, 'path': 'notnews/models/us_not_news.ipynb', 'repo_name': 'notnews/notnews', 'size': '11224'}