<jupyter_start><jupyter_text>Question 2.4<jupyter_code>import numpy as np
import pandas as pd
import re
from collections import Counter
from sklearn.metrics import confusion_matrix
from matplotlib import pyplot as plt<jupyter_output><empty_output><jupyter_text>I decided to use a synthetic dataset I found on Kaggle. The variables were all categorical variables (nominal, ordinal, etc.). For interpretability, decided to use only the nominal features `nom_0` to `nom_4` to predict the `target` variable.<jupyter_code># load data
pd.options.display.max_columns = None
data = pd.read_csv('animal_train.csv')
data = pd.concat([data.loc[:,'nom_0':'nom_4'],data.target],axis=1)
features = data.columns[:-1]
print(data.shape,features)
np.random.seed(2)
indices = np.random.choice(a=len(data.index),size=1000,replace=False) # sample indices for train/valid
train = data.loc[indices[0:800],]
test = data.loc[indices[800:],]
# reindex rows 0 to len-1
train.reset_index(drop=True,inplace=True)
test.reset_index(drop=True,inplace=True)
train.head()
# wrapper/helper functions
def get_priors(labels):
    ''' 
    finds count of each label in labels, converts to probabilities
    returns list of prior probabilities
    '''
    count_dict = Counter(labels) # dict of counts
    counts = list(count_dict.values())
    probs = [c / len(labels) for c in counts]
    prob_dict = dict(zip(list(count_dict.keys()),probs))
    return prob_dict

def get_class_cond(data,label_col,feature_cols,alpha=1):
# def get_class_cond(labels,text):
    '''
    finds class conditional probabilities for all levels of feature_cols 
    and for each class with added psuedocount alpha
    returns dictionary with key = class name; value = class-conditional probabilities
    '''
    classes = np.unique(data[label_col])
    class_cond_dict = {}
    for c in classes:
        class_data = data.loc[data[label_col]==c,]
        nrow = len(class_data.index)
        count_dict = {}
        for f in feature_cols:
            feature_counts = {key:(val + alpha)/nrow for key,val in dict(Counter(class_data[f])).items()} # add pseuedocount
            count_dict.update(feature_counts) # store all feature counts in one dict
        class_cond_dict[c] = count_dict
    return class_cond_dict

def normalize_probs(prob_dict):
    ''' normalize probability to sum to 1 '''
    prob_sums = sum(prob_dict.values())
    return {key:val/prob_sums for key,val in prob_dict.items()}
def get_pred_probs(x_features,prior_probs,class_conds):
    '''
    calculate and return predictive probabilities for example x based on training data, train
    '''
    
    pred_probs = {}
    # for each class calculate predicted probability
    for c in prior_probs:
        pred_prob = prior_probs[c]
        for feature in x_features:
            class_cond_prob = class_conds[c].get(feature)
            pred_prob *= class_cond_prob
        pred_probs[c] = pred_prob
    pred_probs = normalize_probs(pred_probs)
    return pred_probs
def classify(probs):
    return max(probs, key=probs.get)
# get prior and class-conditional of training data
prior_probs = get_priors(train['target'])
class_conds = get_class_cond(train,'target',features)
print(prior_probs)
prob_0 = []; prob_1 = []
pred_class = []; actual_class = test.target
# calculate predictive probabilities for test data
for i,x in test.iterrows():
    x_features = x.loc['nom_0':'nom_4']
    probs = get_pred_probs(x_features,prior_probs,class_conds)
    prob_0.append(probs[0])
    prob_1.append(probs[1])
    pred_class.append(classify(probs))
#     print('predictive probabilities: ', probs)
#     print('predicted class:', classify(probs))
#     print('actual class: ',x.target,'\n')
results = pd.DataFrame(list(zip(prob_0, prob_1,actual_class,pred_class)),
               columns =['P(target=0|X)', 'P(target=1|X)','actual_target','predicted_target'])
results.head()
# check confusion matrix
# print(Counter(results.actual_target),Counter(results.predicted_target))
confusion_mx = confusion_matrix(actual_class,pred_class)
fig, ax = plt.subplots(nrows=1, ncols=1, figsize=[12, 7])
class_labels = ['0','1']
ax.set_xticks(np.arange(len(class_labels)))
ax.set_xticklabels(class_labels, fontsize='15')
ax.set_yticks(np.arange(len(class_labels)))
ax.set_yticklabels(class_labels, fontsize='15')
im = ax.imshow(confusion_mx)
im<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.11702615878843506, 'fraction_numerical': 0.010784763653051858, 'mean_word_length': 4.758256274768824, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '2', 'dir': 'jupyter-structured-clean-dedup', 'id': '211060', 'license': 'MIT', 'n_tokens_mistral': 1462, 'n_tokens_neox': 1421, 'n_words': 331, 'path': 'notebooks/HW4 2.4.ipynb', 'repo_name': 'tylerwu2222/Math156_UCLA_SP21', 'size': '4358'}