<jupyter_start><jupyter_text>Stay classification: QBC evaluation with any stays**02.09.2020**<jupyter_code>import numpy as np
import pandas as pd
import os, sys
sys.path.append('/home/sandm/Notebooks/stay_classification/src/')
%matplotlib inline
import matplotlib.pyplot as plt
from synthetic_data.trajectory_class import get_pickle_trajectory
from synthetic_data.trajectory import get_stay_segs, get_adjusted_stays
from stay_classification.box_classifier.quick_box_classifier import quick_box_method<jupyter_output><empty_output><jupyter_text>Batch data evaluation<jupyter_code>dsec = 1/3600.0
t_total = np.arange(0,24,dsec)
time_thresh = 1/6
dist_thresh=0.25
nr_stays = 3
data_dir = os.path.abspath('../..')+f"/testdata/testdata_training_set__general/"

if os.path.isdir(data_dir):
    print('exists')
        
pkls = [pkl for pkl in os.listdir(data_dir) if 'plots' not in pkl ]   

import glob
data_dir = os.path.abspath('../../')+f"/testdata/testdata_training_set__general/"
os.path.isdir(data_dir)
pkls = glob.glob(data_dir + "*.pkl")
from stay_classification.metrics_etc import eval_synth_data
from stay_classification.metrics import get_segments_scores, get_segments_errs
from stay_classification.metrics_cluster_tools import get_pred_labels, get_labels_from_clusters
from synthetic_data.trajectory import get_stay_indices, get_adjusted_stays

get_err = lambda trues, preds: np.sum(abs(trues-preds))/trues.size<jupyter_output><empty_output><jupyter_text>Load, Classify, Measure<jupyter_code># For the correct nr of stays
lens3 = []
precs3, a_precs3, w_precs3 = [], [], []
recs3, a_recs3, w_recs3 = [], [], []
errs3, a_errs3, w_errs3 = [], [], []

# For the incorrect nr of stays
lens = []
precs, a_precs, w_precs = [], [], []
recs, a_recs, w_recs = [], [], []
errs, a_errs, w_errs = [], [], []


bad_list = []

precrec_limit = 0.80

ii = 0

length_criterion_break = False
iqr_trim = False
verbose = False

total = 1000

for ii in range(0, total):
    
    # Load the data
    trajectory_tag = pkls[ii]   
    path_to_file =  pkls[ii]   #data_dir + trajectory_tag
    t_arr, r_arr, x_arr, segments = get_pickle_trajectory(path_to_file)
    t_segs, x_segs = get_stay_segs(get_adjusted_stays(segments, t_arr))

    # Get the true event indices (needed for the total error)
    true_indices = get_stay_indices(get_adjusted_stays(segments, t_arr), t_arr)
    true_labels = np.zeros(t_arr.shape)
    for pair in true_indices:
        true_labels[pair[0]:pair[1]+1] = 1
    
    # Get the stay clusters
    clusters = quick_box_method(t_arr, x_arr, dist_thresh, time_thresh, 1, False)    
    
    # Make some measurements
    final_len=len(clusters)
    # total scores
    prec, rec, conmat = eval_synth_data(t_arr, segments, clusters)
    # seg. scores
    _, a_prec, w_prec, _, a_rec, w_rec = get_segments_scores(t_arr, segments, clusters)   
    
    # Total error
    pred_labels = get_pred_labels(clusters, t_arr.shape)
    err = get_err(true_labels,pred_labels)
    # Segment errors
    _, a_err, w_err = get_segments_errs(t_arr, segments, clusters)
    
    # Get the expected number of stays (in general)
    stays_tag = int((x_segs.size)/3)
    
    len_all_clusts = len(clusters)
    if final_len != stays_tag:
        lens.append(final_len)
        precs.append(prec)
        a_precs.append(a_prec)
        w_precs.append(w_prec)
        recs.append(rec)        
        a_recs.append(a_rec)        
        w_recs.append(w_rec)
        errs.append(err)        
        a_errs.append(a_err)        
        w_errs.append(w_err)                
    else:
        lens3.append(final_len)
        precs3.append(prec)
        a_precs3.append(a_prec)
        w_precs3.append(w_prec)
        recs3.append(rec)        
        a_recs3.append(a_rec)        
        w_recs3.append(w_rec)        
        errs3.append(err)        
        a_errs3.append(a_err)
        w_errs3.append(w_err)
    
    # progress output
    if ii % int(0.1*total) == 0:
        print(f"{ii:4d} of {total:5d}")
correct_frac = (len(lens3)/total)
incorrect_frac = (len(lens)/total)
print(f"\n    * correct number of stays, {correct_frac:6.3f} ", 
      f"\n        * prec.: {sum(w_precs3)/len(w_precs3):6.3}",
      f"\n        *  rec.: {sum(w_recs3)/len(w_recs3):6.3}",
      f"\n    * incorrect number of stays, {incorrect_frac:6.3f}",
      f"\n        * prec.: {sum(w_precs)/len(w_precs):6.3}",
      f"\n        *  rec.: {sum(w_recs)/len(w_recs):6.3}")<jupyter_output>* correct number of stays,  0.665  
        * prec.:  0.993 
        *  rec.:  0.993 
    * incorrect number of stays,  0.335 
        * prec.:   0.98 
        *  rec.:  0.466<jupyter_text>Visualizations<jupyter_code>from stay_classification.metrics_plotting import plot_scores_stats, plot_errs_stats, plot_scores_stats_cominbed
print(f"Correct stays, {correct_frac:6.3f}; incorrect stays, {incorrect_frac:6.3f}")<jupyter_output>Correct stays,  0.665; incorrect stays,  0.335<jupyter_text>Prec/rec score distributions Total scores<jupyter_code>nr_stays = 'gen_'
title = f"Tot. scores: correct stays, {correct_frac:6.3f}; incorrect stays, {incorrect_frac:6.3f}"

fig, axs = plot_scores_stats(precs3, recs3, precs, recs, title)

fig.savefig("./" + f"metrics__{nr_stays}stays.png")
title = f"Total scores: correct stays, {correct_frac:6.3f}; incorrect stays, {incorrect_frac:6.3f}"

fig, axs = plot_scores_stats_cominbed(precs3, recs3, precs, recs, title)

fig.savefig("./" + f"metrics__{nr_stays}stays_seg_scores_combined_tot.png")<jupyter_output><empty_output><jupyter_text>Segment-averaged score distributions<jupyter_code>title = f"Avg. scores: correct stays, {correct_frac:6.3f}; incorrect stays, {incorrect_frac:6.3f}"

fig, axs = plot_scores_stats(a_precs3, a_recs3, a_precs, a_recs, title)

fig.savefig("./" + f"metrics__{nr_stays}stays_seg_scores_avg.png")
title = f"Avg. Scores: correct stays, {correct_frac:6.3f}; incorrect stays, {incorrect_frac:6.3f}"

fig, axs = plot_scores_stats_cominbed(a_precs3, a_recs3, a_precs, a_recs, title)

fig.savefig("./" + f"metrics__{nr_stays}stays_seg_scores_combined_avg.png")<jupyter_output><empty_output><jupyter_text>Segment, weighted-averaged score distributions<jupyter_code>title = f"W-avg. scores: correct stays, {correct_frac:6.3f}; incorrect stays, {incorrect_frac:6.3f}"

fig, axs = plot_scores_stats(w_precs3, w_recs3, w_precs, w_recs, title)

fig.savefig("./" + f"metrics__{nr_stays}stays_seg_scores_wavg.png")
title = f"W-avg. Scores: correct stays, {correct_frac:6.3f}; incorrect stays, {incorrect_frac:6.3f}"

fig, axs = plot_scores_stats_cominbed(w_precs3, w_recs3, w_precs, w_recs, title)

fig.savefig("./" + f"metrics__{nr_stays}stays_seg_scores_combined_wavg.png")<jupyter_output><empty_output><jupyter_text>Error stats<jupyter_code>title = f"Avg. Error: correct stays, {correct_frac:6.3f}; incorrect stays, {incorrect_frac:6.3f}"

fig, ax = plot_errs_stats(a_errs3, a_errs, title)

fig.savefig("./" + f"metrics__{nr_stays}stays_seg_errs_avg.png")
title = f"W-avg. Error: correct stays, {correct_frac:6.3f}; incorrect stays, {incorrect_frac:6.3f}"

fig, ax = plot_errs_stats(w_errs3, w_errs, title)

fig.savefig("./" + f"metrics__{nr_stays}stays_seg_errs_wavg.png")<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.12687188019966722, 'fraction_numerical': 0.022323904603438714, 'mean_word_length': 4.253459577567371, 'pattern_counts': {'":': 0, '<': 31, '<?xml version=': 0, '>': 31, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '8', 'dir': 'jupyter-structured-clean-dedup', 'id': '79592', 'license': 'MIT', 'n_tokens_mistral': 2928, 'n_tokens_neox': 2837, 'n_words': 488, 'path': 'notebooks/classifiers_playground/quick_box_classifier/classifier_1D__quick_box_classifier__gen_stays_illustrate_metrics.ipynb', 'repo_name': 'm-salewski/stay_classification', 'size': '7212'}