<jupyter_start><jupyter_text>BBC 뉴스 public dataset으로 categorizer 만들기* 아래의 kaggle을 그대로 따라감* https://www.kaggle.com/bbose71/bbc-news-classification<jupyter_code>import os
import sys
import glob
import pandas as pd
import numpy as np
import json
import matplotlib.pyplot as plt
from IPython.core.debugger import set_trace
from IPython.display import display
from pathlib import Path
from smart_open import open
from gensim.utils import simple_preprocess
from tqdm.auto import tqdm
tqdm.pandas()

from sklearn.feature_extraction.text import TfidfVectorizer
from sklearn.feature_selection import chi2
from sklearn.model_selection import cross_val_score, ShuffleSplit, train_test_split
from sklearn.manifold import TSNE
from sklearn.metrics import confusion_matrix

from sklearn.svm import SVC
from sklearn.linear_model import LogisticRegression, SGDClassifier
from sklearn.ensemble import RandomForestClassifier
from sklearn.naive_bayes import MultinomialNB
from sklearn.externals import joblib

import seaborn as sns

%matplotlib inline
data = []
for cat in tqdm(os.listdir('bbc')):
    for fname in glob.glob('bbc/' + cat + '/*.txt'):
        file = Path(fname)
        iid = cat + '-' + file.name.split('.')[0]
        try:
            txt = file.read_text(encoding='latin-1')
        except:
            print(iid)
        data.append([iid, txt, cat])
        
df = pd.DataFrame(data, columns=['ArticleId', 'Text', 'Category']); df
df['category_id'] = df['Category'].factorize()[0]
category_id_df = df[['Category', 'category_id']].drop_duplicates().sort_values('category_id'); category_id_df
category_to_id = dict(category_id_df.values); category_to_id
id_to_category = dict(category_id_df[['category_id', 'Category']].values); id_to_category
df.sample(5, random_state=0)
df.groupby('Category').category_id.count().plot.bar(ylim=0)
tfidf = TfidfVectorizer(sublinear_tf=True, min_df=5, norm='l2', encoding='latin-1', ngram_range=(1,2), stop_words='english')
features = tfidf.fit_transform(df.Text).toarray()
labels = df.category_id; features.shape
N = 3

for Category, category_id in tqdm(sorted(category_to_id.items())):
    features_chi2 = chi2(features, labels==category_id)
    indices = np.argsort(features_chi2[0])
    feature_names = np.array(tfidf.get_feature_names())[indices]
    unigrams = [v for v in feature_names if len(v.split(' '))==1]
    bigrams = [v for v in feature_names if len(v.split(' '))==2]
    print('# {}'.format(Category))
    print('  .Most correlated unigrams:\n    .{}'.format('\n    .'.join(unigrams[-N:])))
    print('  .Most correlated bigrams:\n    .{}'.format('\n    .'.join(bigrams[-N:])))
SAMPLE_SIZE = int(len(features)*0.3)
np.random.seed(0)
indices = np.random.choice(range(len(features)), size=SAMPLE_SIZE, replace=False)
projected_features = TSNE(n_components=2, random_state=0).fit_transform(features[indices]); projected_features.shape
category_id_test = 0
projected_features[labels[indices]==category_id_test]
colors = ['pink', 'green', 'midnightblue', 'orange', 'darkgrey']

for category, category_id in sorted(category_to_id.items()):
    points = projected_features[labels[indices]==category_id]
    plt.scatter(points[:,0], points[:,1], s=30, c=colors[category_id], label=category)
    
plt.title('TF-IDF feature vector for each article\n (projected on 2 dimensions)', fontdict=dict(fontsize=15))
plt.legend();
models = [
    RandomForestClassifier(n_estimators=200, max_depth=3, random_state=0), 
    MultinomialNB(), 
    #SVC(kernel='rbf', probability=True, gamma='auto', C=5), 
    SGDClassifier(random_state=0, max_iter=200, tol=1e-3, loss='log'),  
    LogisticRegression(random_state=0, solver='lbfgs', multi_class='auto')
]
CV = 5
shufflesplit = ShuffleSplit(n_splits=CV, test_size=0.2, random_state=0)
entries = []

for model in tqdm(models):
    model_name = model.__class__.__name__
    accuracies = cross_val_score(model, features, labels, scoring='accuracy', cv=shufflesplit)
    
    for fold_idx, accuracy in enumerate(accuracies):
        entries.append((model_name, fold_idx, accuracy))
cv_df = pd.DataFrame(entries, index=range(CV*len(models)), columns=['model_name', 'fold_idx', 'accuracy']); cv_df
cv_df.groupby('model_name').accuracy.mean()
sns.boxplot(x='model_name', y='accuracy', data=cv_df)
sns.stripplot(x='model_name', y='accuracy', data=cv_df, size=8, jitter=True, edgecolor='gray', linewidth=2)
# model = LogisticRegression(random_state=0, solver='lbfgs', multi_class='auto')
model = SGDClassifier(random_state=0, max_iter=200, tol=1e-3, loss='log')
X_train, X_test, y_train, y_test, indices_train, indices_test = train_test_split(features, labels, df.index, test_size=0.33, random_state=0)
model.fit(X_train, y_train)

y_pred_proba = model.predict_proba(X_test)
y_pred = model.predict(X_test)
X_train.shape, X_test.shape, y_train.shape, y_test.shape
y_pred_proba
conf_mat = confusion_matrix(y_test, y_pred); conf_mat
sns.heatmap(conf_mat, annot=True, fmt='d', xticklabels=category_id_df.Category.values, yticklabels=category_id_df.Category.values)
plt.ylabel('Actual')
plt.xlabel('Predicted')
for predicted in category_id_df.category_id:
    for actual in category_id_df.category_id:
        if predicted != actual and conf_mat[actual, predicted] >= 2:
            print('{} predicted as {}: {} examples'.format(id_to_category[actual], id_to_category[predicted], conf_mat[actual, predicted]))
            display(df.loc[indices_test[(y_test==actual) & (y_pred==predicted)]]['Text'])
model.fit(features, labels)
N = 3

for Category, category_id in tqdm(sorted(category_to_id.items())):
    indices = np.argsort(model.coef_[category_id])
    feature_names = np.array(tfidf.get_feature_names())[indices]
    unigrams = [v for v in feature_names if len(v.split(' '))==1]
    bigrams = [v for v in feature_names if len(v.split(' '))==2]
    print('# {}'.format(Category))
    print('  .Most correlated unigrams:\n    .{}'.format('\n    .'.join(unigrams[-N:])))
    print('  .Most correlated bigrams:\n    .{}'.format('\n    .'.join(bigrams[-N:])))
fnames_test = glob.glob('newsdata/downloaded/*.json')[-20:]

test_data = []
for fname in fnames_test:
    with open(fname, encoding='UTF-8-sig') as f:
        content = json.load(f)
        test_data.append(content['text'])
fnames_test
test_features = tfidf.transform(test_data)
Y_pred = model.predict(test_features); Y_pred
Y_pred_proba = model.predict_proba(test_features).max(axis=1); Y_pred_proba
[id_to_category[cat_id] for cat_id in Y_pred]
model.cat = id_to_category
model.tfidf = tfidf
model_name = 'categorizer.model'
joblib.dump(model, model_name)
_model = joblib.load(model_name)
probas = _model.predict_proba(test_features).max(axis=1); probas
[_model.cat[cat_id] for cat_id in _model.predict(test_features)]
_model = joblib.load(model_name)
df_text = pd.DataFrame(test_data, columns=['text'])
_features = _model.tfidf.transform(df_text.text)
_proba = _model.predict_proba(_features).max(axis=1)
_pred = _model.predict(_features)
_cats = [_model.cat[cat_id] for cat_id in _pred]
df_text['category'] = _cats
df_text['category_proba'] = _proba; df_text
df_text = pd.DataFrame(test_data, columns=['text'])
_features = _model.tfidf.transform(df_text.text)
_proba = _model.predict_proba(_features).max(axis=1)
_pred = _model.predict(_features)
_cats = [_model.cat[cat_id] for cat_id in _pred]
df_text['category'] = _cats
df_text['category_proba'] = _proba; df_text
df_text.text.iloc[2]<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.14360558839333692, 'fraction_numerical': 0.010209564750134336, 'mean_word_length': 5.880776340110906, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 6, 'https://': 1, 'lorem ipsum': 0, 'www.': 1, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 1, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '360290', 'license': 'MIT', 'n_tokens_mistral': 2734, 'n_tokens_neox': 2677, 'n_words': 514, 'path': 'test/BBC news categorizer.ipynb', 'repo_name': 'gem763/crawly', 'size': '7444'}