<jupyter_start><jupyter_text>SynopsisWe attempt a Naive Bayes classifier to predict authorship of the Federalist papers.   Configuration<jupyter_code>base_path = '/Users/rca2t/COURSES/DSI/DS5559/UVA_DSI_REPO'
local_lib = base_path + '/lib'
src_dir = base_path + '/labs/2019-02-28_Lab07/vierthaler-stylometry/fedpapers'<jupyter_output><empty_output><jupyter_text>Libraries<jupyter_code>import glob
import re

import pandas as pd
from numpy.random import randint
import sys; sys.path.append(local_lib)
import textman.textman as tx<jupyter_output><empty_output><jupyter_text>Process  Import raw review data<jupyter_code>files = glob.glob(src_dir+'/*.txt')
corpus = pd.DataFrame([f.replace('.txt','').split('/')[-1].split('_') for f in files], 
                    columns=['chap_id','author'])
corpus['doc_content'] = [open(f, 'r', encoding='utf-8').read() for f in files]
corpus.chap_id = corpus.chap_id.astype('int')
corpus = corpus.set_index('chap_id')
corpus.head()
tokens, vocab = tx.create_tokens_and_vocab(corpus, src_col='doc_content')
tokens = tokens.join(corpus.author)
vocab.head()
tokens.sample(10)
LM = tokens.groupby('author').term_id.value_counts().to_frame().unstack().T.fillna(0)
LM.index = LM.index.droplevel(0)
LMP = (LM + 1) / (LM.sum() + len(vocab)) # Likelihoods
LML = np.log10(LMP[LMP > 0]).fillna(0)
LML.head()
# Priors
# PRIORS = (LM.sum() / LM.sum().sum()).to_frame().rename(columns={0:'p'})
PRIORS = pd.DataFrame(corpus.author.value_counts() / len(corpus))
PRIORS.columns = ['p']
PRIORS['logp'] = np.log10(PRIORS.p)
PRIORS
D = tokens.groupby(['chap_id']).term_id.value_counts().to_frame()\
    .rename(columns={'term_id':'n'}).unstack().fillna(0)
D.columns = D.columns.droplevel(0)
D = D.sort_index()
DBOOL = D.astype('bool').astype('int')
D.head()
results = []
AUTH = 'Unknown'
# AUTH = 'Hamilton'
# AUTH = 'Madison'
# AUTH = 'Jay'

for doc_id in corpus[corpus.author == AUTH].index.values:
    for author in ['Jay', 'Madison', 'Hamilton']:
        x = (DBOOL.loc[doc_id] * LML[author]).sum() + PRIORS.loc[author, 'logp']
        results.append((doc_id, author, x))
        
df = pd.DataFrame(results, columns=['doc_id','author','prop'])
df = df.set_index(['doc_id','author'])
df = df.unstack()
df.columns = df.columns.droplevel(0)
df['prediction'] = df.idxmax(1)
df<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1874192158552348, 'fraction_numerical': 0.014218009478672985, 'mean_word_length': 5.522471910112359, 'pattern_counts': {'":': 0, '<': 13, '<?xml version=': 0, '>': 14, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '3', 'dir': 'jupyter-structured-clean-dedup', 'id': '43775', 'license': 'MIT', 'n_tokens_mistral': 935, 'n_tokens_neox': 911, 'n_words': 148, 'path': 'play/naive-bayes/DS5559_NaiveBayes_Federalist.ipynb', 'repo_name': 'samarthsing/DSI', 'size': '2321'}