<jupyter_start><jupyter_text>Test "best of two" classifier This notebook test a classifier that operates in two layers:- First we use a SVM classifier to label utterances with high degree of certainty.- Afterwards we use heuristics to complete the labeling<jupyter_code>import os
import sys
import pandas as pd
import numpy as np
import random
import pickle
import matplotlib.pyplot as plt

root_path = os.path.dirname(os.path.abspath(os.getcwd()))
sys.path.append(root_path)

from sklearn.svm import SVC
from sklearn.svm import LinearSVC
from sklearn.metrics import classification_report
from sklearn.metrics import confusion_matrix
from src import phase_classification as pc

data_path = os.path.join(root_path,'data')
tables_path = os.path.join(data_path,'tables')
results_path = os.path.join(root_path,'results')
output_path =os.path.join(results_path,'tables')
import importlib
importlib.reload(pc)
WITH_STEMMING = True
#REMOVE_STOPWORDS = True
SEED = 10
NUM_TOPICS = 60
random.seed(SEED)
t = 0
CLASS_W = False
test_i = '[test1]'
file_name = test_i+'IBL_topic_distribution_by_utterance_before_after_{}_{}.xlsx'.format(WITH_STEMMING,NUM_TOPICS)
df_data = pd.read_excel(os.path.join(tables_path,'test','before_after',file_name))
the_keys = list(set(df_data['phase']))
total_samples = 0
class_samples = {}
for key in the_keys:
    n = list(df_data.phase.values).count(key)
    #print("key {}, total {}".format(key,n))
    total_samples += n
    class_samples[key] = n
print(total_samples)
for key in the_keys:
    print("key {}, samples: {}, prop: {}".format(key,class_samples[key],round(class_samples[key]*1.0/total_samples,2)))
filter_rows = list(range(180))+[187,188]
row_label = 180
df_data.head(2)
dfs_all,_ = pc.split_df_discussions(df_data,.0,SEED)
X_all,y_all_1 = pc.get_joined_data_from_df(dfs_all,filter_rows,row_label)
CLASS_W
name_classifier = 'classifier_svm_linear_combination_svc_ba_cw_{}.pickle'.format(CLASS_W)
with open(os.path.join(data_path,name_classifier),'rb') as f:
    svc = pickle.load(f)
    coeff = pickle.load(f)
    t = pickle.load(f)
#t = 0.59
output_first_layer_1 = pc.first_layer_classifier(X_all,t,svc)
comparison = list(zip(output_first_layer_1,y_all_1))
df_data['first_layer'] = output_first_layer_1
second_layer_1 = pc.second_layer_combination_test(X_all,coeff,svc)
second_layer_1.count(-1)
df_data['second_layer'] = second_layer_1
df_data.to_excel(os.path.join(output_path,'[second_layer]'+file_name))
labels = ["Phase {}".format(i) for i in range(1,6)]
df = pd.DataFrame(confusion_matrix(y_all_1, second_layer_1),columns=["Predicted {}".format(i) for i in labels])
df.index = labels
print(classification_report(y_all_1, second_layer_1))
df
print('Accuracy of SVM classifier on training set: {:.2f}'
     .format(svc.score(X_all, y_all_1)))<jupyter_output>Accuracy of SVM classifier on training set: 0.30<jupyter_text>Test 2<jupyter_code>test_i = '[test2]'
file_name = test_i+'IBL_topic_distribution_by_utterance_before_after_{}_{}.xlsx'.format(WITH_STEMMING,NUM_TOPICS)
df_data = pd.read_excel(os.path.join(tables_path,'test','before_after',file_name))
the_keys = list(set(df_data['phase']))
total_samples = 0
class_samples = {}
for key in the_keys:
    n = list(df_data.phase.values).count(key)
    #print("key {}, total {}".format(key,n))
    total_samples += n
    class_samples[key] = n
print(total_samples)
for key in the_keys:
    print("key {}, samples: {}, prop: {}".format(key,class_samples[key],round(class_samples[key]*1.0/total_samples,2)))
dfs_all,_ = pc.split_df_discussions(df_data,.0,SEED)
X_all,y_all_2 = pc.get_joined_data_from_df(dfs_all,filter_rows,row_label)
output_first_layer_2 = pc.first_layer_classifier(X_all,t,name_classifier)
comparison = list(zip(output_first_layer_2,y_all_2))
df_data['first_layer'] = output_first_layer_2
second_layer_2 = pc.second_layer_combination_test(X_all,coeff,svc)
df_data['second_layer'] = second_layer_2
df_data.to_excel(os.path.join(output_path,'[second_layer]'+file_name))
second_layer_2.count(-1)
labels = ["Phase {}".format(i) for i in range(1,6)]
df = pd.DataFrame(confusion_matrix(y_all_2, second_layer_2),columns=["Predicted {}".format(i) for i in labels])
df.index = labels
print(classification_report(y_all_2, second_layer_2))
df
print('Accuracy of SVM classifier on training set: {:.2f}'
     .format(svc.score(X_all, y_all_2)))
y_all = y_all_1+y_all_2
pred = second_layer_1 + second_layer_2
df = pd.DataFrame(confusion_matrix(y_all, pred),columns=["Predicted {}".format(i) for i in labels])
df.index = labels
print(classification_report(y_all, pred))
df
print("Accuracy {0:.3f}".format(np.sum(confusion_matrix(y_all, pred).diagonal())/len(y_all)))
bs = [pc.unit_vector(x) for x in y_all]
y_pred = [pc.unit_vector(x) for x in pred]
np.sqrt(np.sum([np.square(y_pred[i]-bs[i]) for i in range(len(y_all))])/(len(y_all)*2))<jupyter_output>Accuracy 0.466
{'fraction_non_alphanumeric': 0.13732756845789582, 'fraction_numerical': 0.0168828494955734, 'mean_word_length': 7.478184991273997, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '2', 'dir': 'jupyter-structured-clean-dedup', 'id': '1788', 'license': 'MIT', 'n_tokens_mistral': 1903, 'n_tokens_neox': 1842, 'n_words': 329, 'path': 'notebooks/3-. Check combination 2 layers classifier with test set [utterance level][180t].ipynb', 'repo_name': 'cinai/classification_ibl', 'size': '4857'}