<jupyter_start><jupyter_text>Test svm classifier This notebook test a classifier that operates in two layers:- First we use a SVM classifier to label utterances with high degree of certainty.- Afterwards we use heuristics to complete the labeling<jupyter_code>import os
import sys
import pandas as pd
import numpy as np
import random
import pickle
import matplotlib.pyplot as plt

root_path = os.path.dirname(os.path.abspath(os.getcwd()))
sys.path.append(root_path)

from sklearn.svm import SVC
from sklearn.svm import LinearSVC
from sklearn.metrics import classification_report
from sklearn.metrics import confusion_matrix
from src import phase_classification as pc

data_path = os.path.join(root_path,'data')
tables_path = os.path.join(data_path,'tables')
results_path = os.path.join(root_path,'results')
output_path =os.path.join(results_path,'tables')
import importlib
importlib.reload(pc)
WITH_STEMMING = True
#REMOVE_STOPWORDS = True
SEED = 10
NUM_TOPICS = 60
random.seed(SEED)
test_i = '[test1]'
file_name = test_i+'IBL_topic_distribution_by_utterance_with_phrase_before_and_after_time_utterance_minimum_0_words_with_stemming_{}_{}.xlsx'.format(WITH_STEMMING,NUM_TOPICS)
df_data = pd.read_excel(os.path.join(tables_path,'test',file_name))
with open(os.path.join(data_path,'random_training.pickle'),'rb') as f:
    proportions = pickle.load(f)
    step_1 = pickle.load(f)
    step_2 = pickle.load(f)    
dummy_phase = np.argmax(proportions)+1
the_keys = list(set(df_data['phase']))
total_samples = 0
class_samples = {}
for key in the_keys:
    n = list(df_data.phase.values).count(key)
    #print("key {}, total {}".format(key,n))
    total_samples += n
    class_samples[key] = n
print(total_samples)
for key in the_keys:
    print("key {}, samples: {}, prop: {}".format(key,class_samples[key],round(class_samples[key]*1.0/total_samples,2)))
filter_rows = list(range(0,180))+[187,188]
filter_labels = [60]
all_set = pc.split_df_test(df_data)
X_all_1,y_all_1 = pc.get_data_from_dict(all_set,filter_rows)
print(len(y_all_1))
output_first_layer_1 = [dummy_phase for i in y_all_1]
labels = ["Phase {}".format(i) for i in range(1,6)]
df = pd.DataFrame(confusion_matrix(y_all_1, output_first_layer_1),columns=["Predicted {}".format(i) for i in labels])
df.index = labels
print(classification_report(y_all_1, output_first_layer_1))
df
print("Accuracy {0:.3f}".format(np.sum(confusion_matrix(y_all_1, output_first_layer_1).diagonal())/len(y_all_1)))
bs = [pc.unit_vector(x) for x in y_all_1]
y_pred = [pc.unit_vector(x) for x in output_first_layer_1]
np.sqrt(np.sum([np.square(y_pred[i]-bs[i]) for i in range(len(y_all_1))])/(len(y_all_1)*2))<jupyter_output><empty_output><jupyter_text>Test 2<jupyter_code>test_i = '[test2]'
file_name = test_i+'IBL_topic_distribution_by_utterance_with_phrase_before_and_after_time_utterance_minimum_0_words_with_stemming_{}_{}.xlsx'.format(WITH_STEMMING,NUM_TOPICS)
df_data = pd.read_excel(os.path.join(tables_path,'test',file_name))
the_keys = list(set(df_data['phase']))
total_samples = 0
class_samples = {}
for key in the_keys:
    n = list(df_data.phase.values).count(key)
    #print("key {}, total {}".format(key,n))
    total_samples += n
    class_samples[key] = n
print(total_samples)
for key in the_keys:
    print("key {}, samples: {}, prop: {}".format(key,class_samples[key],round(class_samples[key]*1.0/total_samples,2)))
all_set = pc.split_df_test(df_data)
X_all_2,y_all_2 = pc.get_data_from_dict(all_set,filter_rows)
t = 0.00
output_first_layer_2 = [dummy_phase for i in y_all_2]
labels = ["Phase {}".format(i) for i in range(1,6)]
df = pd.DataFrame(confusion_matrix(y_all_2, output_first_layer_2),columns=["Predicted {}".format(i) for i in labels])
df.index = labels
print(classification_report(y_all_2, output_first_layer_2))
df
print("Accuracy {0:.3f}".format(np.sum(confusion_matrix(y_all_2, output_first_layer_2).diagonal())/len(y_all_2)))
bs = [pc.unit_vector(x) for x in y_all_2]
y_pred = [pc.unit_vector(x) for x in output_first_layer_2]
np.sqrt(np.sum([np.square(y_pred[i]-bs[i]) for i in range(len(y_all_2))])/(len(y_all_2)*2))
y_all = y_all_1+y_all_2
pred = output_first_layer_1 + output_first_layer_2
df = pd.DataFrame(confusion_matrix(y_all, pred),columns=["Predicted {}".format(i) for i in labels])
df.index = labels
print(classification_report(y_all, pred))
df
print("Accuracy {0:.3f}".format(np.sum(confusion_matrix(y_all, pred).diagonal())/len(y_all)))
bs = [pc.unit_vector(x) for x in y_all]
y_pred = [pc.unit_vector(x) for x in pred]
np.sqrt(np.sum([np.square(y_pred[i]-bs[i]) for i in range(len(y_all))])/(len(y_all)*2))<jupyter_output>Accuracy 0.349
{'fraction_non_alphanumeric': 0.14186701321204245, 'fraction_numerical': 0.018843404808317088, 'mean_word_length': 7.320720720720721, 'pattern_counts': {'":': 0, '<': 8, '<?xml version=': 0, '>': 8, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '2', 'dir': 'jupyter-structured-clean-dedup', 'id': '84610', 'license': 'MIT', 'n_tokens_mistral': 1834, 'n_tokens_neox': 1781, 'n_words': 323, 'path': 'notebooks/3-- Check random test set.ipynb', 'repo_name': 'cinai/classification_ibl', 'size': '4617'}