<jupyter_start><jupyter_text>Load and select datasf_use have 20000 samples<jupyter_code>import sframe
import pandas as pd
import numpy as np
sf = sframe.load_sframe('./data_sf/text_receiver_sender_small')
sf.show
#sf_use = sf.sample(.1, seed=5)
#sf = sf_use.save('./data_sf/text_receiver_sender_small')
sf_use = sf.to_dataframe()
sf_use.shape
from sklearn.cross_validation import train_test_split

sf_train, sf_test = train_test_split(sf_use,  test_size = 0.2)
print('Sample number of train and test are: %d, %d' % (len(sf_train), len(sf_test)))<jupyter_output>Sample number of train and test are: 17360, 4341<jupyter_text>Seperate test to known label and unknown label<jupyter_code>unique_train_senders = np.unique(sf_train['senders'])
unique_train_receivers = np.unique(sf_train['receivers'])

sf_test.loc[:, 'known_s'] = sf_test['senders'].apply(lambda x: 1 if x in unique_train_senders else 0)
sf_test.loc[:, 'known_r'] = sf_test['receivers'].apply(lambda x: 1 if x in unique_train_receivers else 0)
sf_test.head()
print('Senders:')
print('Number of known senders in train data: %d' % (np.unique(sf_train['senders']).shape[0]))
print('Number of known senders in test data: %d' % (np.unique(sf_test['senders'])).shape[0])
print('Number of samples with known sender in test data: %d' % (sf_test['known_s'].sum()))
print('Percentage of samples with known senders in test data: %.3f'% (float(sf_test['known_s'].sum()) / sf_test.shape[0]))

print ''
print('Receivers:')
print('Number of known receivers in train data: %d' % (np.unique(sf_train['receivers']).shape[0]))
print('Number of known receivers in test data: %d' % (np.unique(sf_test['receivers'])).shape[0])
print('Number of samples with known receivers in test data: %d' % (sf_test['known_r'].sum()))
print('Percentage of samples with known receivers in test data: %.3f'% (float(sf_test['known_r'].sum()) / sf_test.shape[0]))
(sf_test['known_s']==1).mean()<jupyter_output><empty_output><jupyter_text>Only looking at the samples with known labels:<jupyter_code>sf_test_known_s = sf_test[sf_test['known_s']==1]
sf_test_known_r = sf_test[sf_test['known_r']==1]
#import cPickle as pickle

from sklearn.feature_extraction.text import CountVectorizer
from sklearn.feature_extraction.text import TfidfVectorizer
from sklearn.preprocessing import LabelEncoder
from sklearn.naive_bayes import MultinomialNB
from sklearn.metrics import f1_score, accuracy_score, precision_score
import time

import pylab as plt
%matplotlib inline


sv = LabelEncoder()
rv = LabelEncoder()
dv = TfidfVectorizer(max_df=0.06, min_df=0.001)<jupyter_output><empty_output><jupyter_text>label all known sender and receiver in sf_use<jupyter_code>sv.fit(sf_use['senders']) # label all known sender in sf_use
rv.fit(sf_use['receivers']) # label all known receivers in sf_use
sv.classes_.shape
rv.classes_.shape
train_X = dv.fit_transform(sf_train['text'])
train_s = sv.transform(sf_train['senders']) 
train_r = rv.transform(sf_train['receivers']) 
val_X_s = dv.transform(sf_test_known_s['text'])
val_s = sv.transform(sf_test_known_s['senders'])

val_X_r = dv.transform(sf_test_known_r['text'])
val_r = rv.transform(sf_test_known_r['receivers'])
train_X.shape
val_X_s.shape
val_X_r.shape<jupyter_output><empty_output><jupyter_text>double check if all the labels in val exist in train labels (should all included)<jupyter_code>l3 = np.array([x for x in val_s if x not in train_s])
1 - float(l3.shape[0]) / val_s.shape[0]
train_s.shape
u, c = np.unique(train_s, return_counts=True)
indices = np.argsort(c)[::-1]
fig = plt.scatter(range(u.shape[0]), c[indices])
plt.yscale('log')

u, c = np.unique(train_r, return_counts=True)
indices = np.argsort(c)[::-1]
fig = plt.scatter(range(u.shape[0]), c[indices])
plt.yscale('log')
train_X.shape
count_train_X = (train_X.toarray()).sum(axis = 0) # note that train_X is a sparse matrix, we change it to an array before plot
type(train_X)
fig = plt.hist(count_train_X, 50)
plt.yscale('log')
count_train_X[0:10]
count_train_X.max()
count_train_X.max()/13575
clf_s = MultinomialNB(alpha=0.0001, fit_prior=False)
clf_s.fit(train_X, train_s)
pred_s = clf_s.predict(val_X_s)
print f1_score(val_s, pred_s, average='micro')
print f1_score(val_s, pred_s, average='macro')
print f1_score(val_s, pred_s, average='weighted')
print precision_score(val_s, pred_s, average='weighted')
pred_s
val_s
sv.inverse_transform(1360)
clf_r = MultinomialNB(alpha=0.0001, fit_prior=False)
clf_r.fit(train_X, train_r)
pred_r = clf_r.predict(val_X_r)
print f1_score(val_r, pred_r, average='micro')
print f1_score(val_r, pred_r, average='macro')
print f1_score(val_r, pred_r, average='weighted')
print precision_score(val_r, pred_r, average='weighted')<jupyter_output>0.782427745665
0.370613760792
0.805145634982
0.85529063531<jupyter_text>Test some other classifiers<jupyter_code># Not working
#from sklearn.linear_model import LogisticRegression
#clf_s = LogisticRegression(n_jobs=-1)
#clf_s.fit(train_X, train_s)
#pred_s = clf_s.predict(val_X_s)
#print f1_score(val_s, pred_s, average='micro')
#print f1_score(val_s, pred_s, average='macro')
#print f1_score(val_s, pred_s, average='weighted')
#print precision_score(val_s, pred_s, average='weighted')<jupyter_output><empty_output><jupyter_text>Check probability output  For sender:<jupyter_code>pred_prob_s = clf_s.predict_proba(val_X_s)
pred_prob_s.shape
pred_prob_s[0]
pred_prob_s[0].max()
clf_s.classes_<jupyter_output><empty_output><jupyter_text>Now, take a look for the unknown label test:<jupyter_code>sf_test_unknown_s = sf_test[sf_test['known_s']==0]
val_X_unk_s = dv.transform(sf_test_unknown_s['text'])
val_unk_s = sv.transform(sf_test_unknown_s['senders'])
pred_unk_s = clf_s.predict(val_X_unk_s)
print f1_score(val_unk_s, pred_unk_s, average='micro')
print f1_score(val_unk_s, pred_unk_s, average='macro')
print f1_score(val_unk_s, pred_unk_s, average='weighted')
print precision_score(val_unk_s, pred_unk_s, average='weighted')
clf_s = MultinomialNB(alpha=0.1, fit_prior=False)
clf_s.fit(train_X, train_s)
pred_s = clf_s.predict(val_X_s)
print f1_score(val_s, pred_s, average='micro')
print f1_score(val_s, pred_s, average='macro')
print f1_score(val_s, pred_s, average='weighted')
print precision_score(val_s, pred_s, average='weighted')

pred_prob_s = clf_s.predict_proba(val_X_s)
pred_prob_unk_s = clf_s.predict_proba(val_X_unk_s)


plt.figure(figsize=(16,4))
plt.subplot(1, 2, 1)
plt.hist(pred_prob_s.max(axis=1),20)
plt.title('Histogram of predicted maximun probabilities for \n sender with known labels in training (4087 sampes) \n alpha=0.1, acc=0.94')
#plt.yscale('log')

plt.subplot(1, 2, 2)
plt.hist(pred_prob_unk_s.max(axis=1),20)
plt.title('Histogram of predicted maximun probabilities for \n sender with UNKNOWN labels in training (254 sampes) \n alpha=0.1, acc=0.0')
#plt.yscale('log')<jupyter_output>0.945906432749
0.826037925229
0.930687653983
0.924209846434
{'fraction_non_alphanumeric': 0.13609898107714702, 'fraction_numerical': 0.033478893740902474, 'mean_word_length': 8.04078947368421, 'pattern_counts': {'":': 0, '<': 30, '<?xml version=': 0, '>': 30, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '8', 'dir': 'jupyter-structured-clean-dedup', 'id': '488047', 'license': 'MIT', 'n_tokens_mistral': 2852, 'n_tokens_neox': 2691, 'n_words': 539, 'path': 'Step1-SimpleNBModel.ipynb', 'repo_name': 'liufuyang/TS_ML_training_SR_classification', 'size': '6870'}