<jupyter_start><jupyter_text>See instructions here:https://github.com/tapilab/elevate-osna-starter/blob/master/lessons/week2/README.mdday-2<jupyter_code>from collections import Counter
import numpy as np
import pandas as pd
import re
from sklearn.linear_model import LogisticRegression
from sklearn.feature_extraction import DictVectorizer
from sklearn.feature_extraction.text import CountVectorizer
def load_data(datafile):
    """
    Read your data into a single pandas dataframe where
    - each row is an instance to be classified
    (this could be a tweet, user, or news article, depending on your project)
    - there is a column called `label` which stores the class label (e.g., the true
      category for this row)
    """
    df = pd.read_csv(datafile)[['text', 'hostile']]
    df.columns = ['text', 'label']
    df['label'] = ['hostile' if i==1 else 'nonhostile' for i in df.label]
    return df

df = load_data('~/Dropbox/elevate/harassment/training_data/data.csv.gz')
df.head()
# what is the distribution over class labels?
df.label.value_counts()
def make_features(df):
    vec = DictVectorizer()
    feature_dicts = []
    # just as an initial example, we will consider three
    # word features in the model.
    words_to_track = ['you', 'hate', 'love']
    # will get different model for different features.
    #words_to_track = ['you']
    for i, row in df.iterrows():
        features = {}
        token_counts = Counter(re.sub('\W+', ' ', row['text'].lower()).split())
        for w in words_to_track:
            features[w] = token_counts[w]
        feature_dicts.append(features)
    X = vec.fit_transform(feature_dicts)
    return X, vec
                
X, vec = make_features(df)
X
# https://docs.scipy.org/doc/scipy/reference/generated/scipy.sparse.csr_matrix.html
# Now, use CountVectorizer to create a term feature matrix.
count_vec = CountVectorizer()
X_words = count_vec.fit_transform(df.text)
X_words.shape
# how sparse is it? 
def print_sparsity(X_words):
    print('%d words' % X_words.shape[1])
    num_cells = X_words.shape[0] * X_words.shape[1]
    print('%d of %d possible cells are non-zero (%.2f%%)' %
          (X_words.nnz, num_cells,
           100 * X_words.nnz/num_cells))
print_sparsity(X_words)
# how does sparsity vary with min_df?
for min_df in [1,2,5,10]:
    count_vec = CountVectorizer(min_df=min_df)
    print('\n\nmin_df=%d' % min_df)
    X_words = count_vec.fit_transform(df.text)
    print_sparsity(X_words)
# top terms?
def print_top_words(X_words, count_vec, n=10):
    features = count_vec.get_feature_names()
    word_counts = X_words.sum(axis=0).A1
    for i in np.argsort(word_counts)[::-1][:n]:
        print('%20s\t%d' % (features[i], word_counts[i]))

print_top_words(X_words, count_vec)
# top terms using different ngrams
for ngram in [(1,1), (2,2), (3,3), (1,3)]:
    count_vec = CountVectorizer(min_df=2, ngram_range=ngram)
    print('\nngram_range=%s' % str(ngram))
    X_words = count_vec.fit_transform(df.text)
    print_sparsity(X_words)
    print_top_words(X_words, count_vec)
# we'll first store the classes separately in a numpy array
y = np.array(df.label)
Counter(y)
# store the class names
class_names = set(df.label)
# how often does each word appear in each class?
for word, idx in list(vec.vocabulary_.items())[:10]:
    for class_name in class_names:
        class_idx = np.where(y==class_name)[0]
        print('%20s\t%20s\t%d' % (word, class_name, X[class_idx, idx].sum()))
# to combine two sparse matrices:
from scipy.sparse import csr_matrix, hstack # "horizontal stack"
m1 = csr_matrix([[1,2,3], [4,5,6]])
print('m1')
print(m1.todense())
m2 = csr_matrix([[7,8], [9,10]])
print('m2')
print(m2.todense())
hstack([m1, m2]).todense()
X_all = hstack([X, X_words])
X_all.shape
# fit a LogisticRegression classifier.
clf = LogisticRegression(solver='lbfgs', multi_class='auto')
clf.fit(X_all, y)
# for binary classification, LogisticRegression stores a single coefficient vector
clf.coef_
# this would be a matrix for a multi-class probem.
# for binary classification, the coefficients for the negative class is just the negative of the positive class.
coef = [-clf.coef_[0], clf.coef_[0]]
print(coef)
clf.classes_
from sklearn.model_selection import KFold
from sklearn.metrics import accuracy_score

kf = KFold(n_splits=5, shuffle=True, random_state=42)
accuracies = []
for train, test in kf.split(X):
    clf.fit(X[train], y[train])
    pred = clf.predict(X[test])
    accuracies.append(accuracy_score(y[test], pred))
    
    
print('accuracy over all cross-validation folds: %s' % str(accuracies))
print('mean=%.2f std=%.2f' % (np.mean(accuracies), np.std(accuracies)))<jupyter_output>accuracy over all cross-validation folds: [0.5424354243542435, 0.5476014760147602, 0.5276752767527675, 0.5402214022140222, 0.5236336779911374]
mean=0.54 std=0.01
{'fraction_non_alphanumeric': 0.1307405875051717, 'fraction_numerical': 0.03206454282167977, 'mean_word_length': 4.330760749724366, 'pattern_counts': {'":': 0, '<': 4, '<?xml version=': 0, '>': 4, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '420388', 'license': 'MIT', 'n_tokens_mistral': 1816, 'n_tokens_neox': 1734, 'n_words': 430, 'path': 'notebooks/W2L2.ipynb', 'repo_name': 'tapilab/elevate-osna-news', 'size': '4834'}