<jupyter_start><jupyter_text>Train classifier by layersThis notebook trains a classifier that operates in two layers:- First we use a SVM classifier to label utterances with high degree of certainty.- Afterwards we use heuristics to complete the labeling  Import and path definition<jupyter_code>import os
import pandas as pd
import numpy as np
import random
import pickle
import sys
import matplotlib.pyplot as plt

root_path = os.path.dirname(os.path.dirname(os.path.abspath(os.getcwd())))
sys.path.append(root_path)

from sklearn.svm import SVC
from sklearn.svm import LinearSVC
from sklearn.metrics import classification_report
from sklearn.metrics import confusion_matrix
from src import phase_classification as pc

data_path = os.path.join(root_path,'data')
tables_path = os.path.join(data_path,'tables')<jupyter_output><empty_output><jupyter_text>Load data<jupyter_code>WITH_STEMMING = True
#REMOVE_STOPWORDS = True
SEED = 10
NUM_TOPICS = 60
random.seed(SEED)
CLASS_W = False
file_name = '[train]IBL_topic_distribution_by_utterance_with_phrase_before_and_after_time_utterance_minimum_0_words_with_stemming_{}_{}.xlsx'.format(WITH_STEMMING,NUM_TOPICS)
df_data = pd.read_excel(os.path.join(tables_path,'train',file_name))<jupyter_output><empty_output><jupyter_text>Data description<jupyter_code>df_data.head()
the_keys = list(set(df_data['phase']))
total_samples = 0
class_samples = {}
for key in the_keys:
    n = list(df_data.phase.values).count(key)
    #print("key {}, total {}".format(key,n))
    total_samples += n
    class_samples[key] = n
print(total_samples)
proportions = []
for key in the_keys:
    proportion = round(class_samples[key]*1.0/total_samples,2)    
    print("key {}, samples: {}, prop: {}".format(key,class_samples[key],proportion))<jupyter_output>1600
key 1, samples: 403, prop: 0.25
key 2, samples: 175, prop: 0.11
key 3, samples: 406, prop: 0.25
key 4, samples: 62, prop: 0.04
key 5, samples: 554, prop: 0.35<jupyter_text>split data<jupyter_code>filter_rows = list(range(180))+[187,188]
row_label = 180
dfs_train,dfs_val = pc.split_df_discussions(df_data,.2,SEED)
X_train,y_train = pc.get_joined_data_from_df(dfs_train,filter_rows,row_label)
X_val,y_val = pc.get_joined_data_from_df(dfs_val,filter_rows,row_label)
len(X_train)
dfs_all,_ = pc.split_df_discussions(df_data,.0,SEED)
X_all,y_all = pc.get_joined_data_from_df(dfs_all,filter_rows,row_label)<jupyter_output><empty_output><jupyter_text>Classify first layer<jupyter_code>len(X_train[0])
class_weight = {}
for key in the_keys:
    class_weight[key] = 1000.0/class_samples[key]<jupyter_output><empty_output><jupyter_text>Note that if you remove class_weight then the accuracy is up to 0.5<jupyter_code>if CLASS_W:
    svc = SVC(kernel='linear',random_state=SEED,probability=True,class_weight=class_weight)
else:
    svc = SVC(kernel='linear',random_state=SEED,probability=True)
svc.fit(X_train, y_train)
print('Accuracy of SVM classifier on training set: {:.2f}'
     .format(svc.score(X_train, y_train)))
print('Accuracy of SVM classifier on test set: {:.2f}'
     .format(svc.score(X_val, y_val)))
pred = svc.predict(X_val)
labels = ["Phase {}".format(i) for i in range(1,6)]
df = pd.DataFrame(confusion_matrix(y_val, pred),columns=["Predicted {}".format(i) for i in labels])
df.index = labels
#print(" ")
print(classification_report(y_val, pred))
df
print('Accuracy of SVM classifier on training set: {:.2f}'
     .format(svc.score(X_all, y_all)))
pred_val = svc.predict_proba(X_all)
output_first_layer = [np.argmax(pred_val[i])+1 for i in range(len(pred_val))]
df_data['first_layer'] = output_first_layer
df_data.head()
df_data.to_excel(os.path.join(tables_path,'[one_layer][cw_{}]'.format(CLASS_W)+file_name))
with open(os.path.join(data_path,'classifier_svm_one_layer_cw_{}.pickle'.format(CLASS_W)),'wb') as f:
    pickle.dump(svc,f)
df = pd.DataFrame(confusion_matrix(y_all, output_first_layer),columns=["Predicted {}".format(i) for i in labels])
df.index = labels
print(classification_report(y_all, output_first_layer))
df
print('Accuracy of SVM classifier on training set: {:.2f}'
     .format(svc.score(X_all, y_all)))<jupyter_output>Accuracy of SVM classifier on training set: 0.51
{'fraction_non_alphanumeric': 0.12673362027737925, 'fraction_numerical': 0.018890483022477284, 'mean_word_length': 7.299603174603175, 'pattern_counts': {'":': 0, '<': 23, '<?xml version=': 0, '>': 23, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '6', 'dir': 'jupyter-structured-clean-dedup', 'id': '398986', 'license': 'MIT', 'n_tokens_mistral': 1639, 'n_tokens_neox': 1543, 'n_words': 307, 'path': 'notebooks/2. Train classifier (one layer)/2-. Train classifier one layer [utterance level][180t].ipynb', 'repo_name': 'cinai/classification_ibl', 'size': '4182'}