<jupyter_start><jupyter_text>MuTraff - Analyze Single ExperimentTakes a single experiment results files and makes analitical diagnosis.Use files: edge_stats.csv<jupyter_code>%matplotlib inline

import os
import re
import pprint as pp
import pandas as pd
import numpy as np

import matplotlib.pyplot as plt
import matplotlib.patches as mpatches
import matplotlib.mlab as mlab
import matplotlib.lines as mlines
import matplotlib as mpl
from matplotlib.cm import get_cmap

from IPython.display import display
from IPython.display import HTML
import IPython.core.display as di

#plt.style.use('ggplot')
MUTRAFF_HOME="/Users/alvaro/Desktop/workspace/mutraff/uah-gist-mutraff-bastra"
MUTRAFF_EXP_PATH=MUTRAFF_HOME + "/experiments/tmp"
SIMUL_TIME_STATS=None
COLUMNS=[]
exp={}
label={}

SMALL_FONT_SIZE=10
MID_FONT_SIZE=12
BIG_FONT_SIZE=14<jupyter_output><empty_output><jupyter_text>Load data from experiment file<jupyter_code>def load_data(experiment):
    global MUTRAFF_EXP_PATH
    filename = "{}/{}/{}".format(MUTRAFF_EXP_PATH,experiment,"edge_stats.csv")
    print("Parsing file "+filename)
    df = pd.read_csv(filename)
    return df<jupyter_output><empty_output><jupyter_text>Time graphDraw a time graph for the defined measure in time<jupyter_code># Example: 
# draw_graph( data, "Total halted vehicles", "traf_halted_veh_num", "Halted vehicles", "sum" )
def create_graph( title, measure_name, measure_label ):

    # Prepare the plot
    fig, axes = plt.subplots(figsize=(15,7))

    plt.xlabel('Simulation time', fontsize=BIG_FONT_SIZE)
    plt.ylabel(measure_label,fontsize=BIG_FONT_SIZE)

    # BLACK STYLE
    axes.spines['bottom'].set_color('0.5')
    axes.spines['top'].set_color('0.5')
    axes.spines['right'].set_color('0.5')
    axes.spines['left'].set_color('0.5')
    axes.patch.set_facecolor('0.1')
    plt.grid(b=True, which='major', color='0.2', linestyle='-')
    plt.grid(b=True, which='minor', color='0.2', linestyle='-')
    axes.tick_params(axis='x', colors='0.7', which='both')
    axes.tick_params(axis='y', colors='0.7', which='both')
    axes.yaxis.label.set_color('0.9')
    axes.xaxis.label.set_color('0.9')
    axes.margins(0.5)
    plt.title( title, color='0.7' )
    fig.patch.set_facecolor('0.15')

        
    # WHILE STYLE
    axes.spines['bottom'].set_color('0')
    axes.spines['top'].set_color('0')
    axes.spines['right'].set_color('0')
    axes.spines['left'].set_color('0')
    axes.patch.set_facecolor('1')
    plt.grid(b=True, which='major', color='0.2', linestyle='-')
    plt.grid(b=True, which='minor', color='0.2', linestyle='-')
    axes.tick_params(axis='x', colors='0.7', which='both')
    axes.tick_params(axis='y', colors='0.7', which='both')
    axes.yaxis.label.set_color('0.2')
    axes.xaxis.label.set_color('0.2')
    axes.margins(0.5)
    axes.set_xlim( xmin=0, xmax=3000 )
    plt.title( title, color='0.2' )
    fig.patch.set_facecolor('1')
    
    plt.rc('font', size=MID_FONT_SIZE)          # controls default text sizes
    plt.rc('axes', titlesize=MID_FONT_SIZE)     # fontsize of the axes title
    plt.rc('axes', labelsize=BIG_FONT_SIZE)    # fontsize of the x and y labels
    plt.rc('xtick', labelsize=SMALL_FONT_SIZE)    # fontsize of the tick labels
    plt.rc('ytick', labelsize=SMALL_FONT_SIZE)    # fontsize of the tick labels
    plt.rc('legend', fontsize=BIG_FONT_SIZE)    # legend fontsize
    plt.rc('figure', titlesize=BIG_FONT_SIZE)  # fontsize of the figure title
    
    return fig, axes
# Example: 
# draw_graph( data, "Total halted vehicles", "traf_halted_veh_num", "Halted vehicles", "sum" )
def draw_graphs( exp_list, title, measure_name, measure_label, function ):
    # exps = [ {'exp':experiment, 'data':data1}, ... ]

    # Rolling mean window size
    rolling_width=10
    flag_rolling = False

    # Prepare the plot
    fig, axes = create_graph( title, measure_name, measure_label )

    colors=[ 'red', 'green', 'blue', 'orange', 'black', 'pink', 'brown']

#--- Calculate y_min , y_max -------------------------
#    ax_y_min, ax_y_max = 1000, -1000
#    for item in df_list:
#        min, max = np.min(item['data']), np.max(item['data'])
#        if min < ax_y_min:
#            ax_y_min = min
#        if max > ax_y_max:
#            ax_y_max = max
#    axes.set_ylim(ax_y_min,ax_y_max])
#----------------------------------------------------
    my_handles = []
    
    color_idx = 0
    for exp in exp_list:
        df = exp['data']
        if ( function == "sum" ):
            ds  = df.groupby(['time']).sum().unstack()[measure_name]
            if flag_rolling:
                ds_rolling=df.groupby(['time']).rolling(rolling_width).sum().unstack()[measure_name]
        if ( function == "count" ):
            ds = df.groupby(['time']).count().unstack()[measure_name]
            if flag_rolling:
                ds_rolling=df.groupby(['time']).rolling(rolling_width).count().unstack()[measure_name]
                #ds = exp[exp[measure_name]>0 ].groupby(['time']).count().unstack()[measure_name]
        if ( function == "mean" ):
            ds = df.groupby(['time']).mean().unstack()[measure_name]
            if flag_rolling:
                ds_rolling=df.groupby(['time']).rolling(rolling_width).mean().unstack()[measure_name]
    
        # Draw
        my_handles.append( mpatches.Patch(color=colors[color_idx], label=exp['label']))
        ax1 = ds.plot(ax=axes, color=colors[color_idx])
        color_idx=color_idx+1
        # ax1 = ds.plot(ax=axes,color="blue")
        # ds_rolling.plot(ax=ax1,color="red")
        
    axes.legend(handles=my_handles, loc='lower center', fancybox=True, framealpha=0.5, fontsize=BIG_FONT_SIZE)
    # plt.rc('legend', fontsize=30)
    #params = {'legend.fontsize': 12, 'legend.handlelength': 2}
    #plt.rcParams.update(params)<jupyter_output><empty_output><jupyter_text>Analyze experiment<jupyter_code>def analyze_experiments( exp_list ):
    # display(HTML("<h2>Experiments results for {}</h2>".format(exp_list)))
    display(HTML("<h2>Experiments results</h2>"))
    display(HTML("<style>div.output_subarea table {font-size: 10px;}</style>"))
    # print( "Columnas disponibles {}".format(df.columns) )
    
    draw_graphs( exps, "Total halted vehicles", "traf_halted_veh_num", "Halted vehicles", "sum" )
    draw_graphs( exps, "Mean speed", "traf_av_speed", "Mean speed", "mean" )
    
    exp_list_1 = []
    for item in exp_list:
        df=item['data']
        exp_list_1.append( { 'exp':item['exp'], 'data':df[df["traf_halted_veh_num"]>0 ], 'label':item['label'] } )
    # exps1 = [ {'exp':exp_name, 'data':df[df["traf_halted_veh_num"]>0 ]} ]
    draw_graphs( exp_list_1, "Number of congested edges", "traf_halted_veh_num", "Number of edges", "count" )<jupyter_output><empty_output><jupyter_text>Experiments<jupyter_code>exp[0]="alcalahenares_M_mutraff_tele60_uni5x16_timeALL_fulltraffic_logit20_11_190508_003728"


exp[2200]="alcalahenares_M3h_nomaps_tele60_timeALL_fulltraffic_190510_235642"
exp[2201]="alcalahenares_M3h_mutraff_tele60_uni5x16_timeALL_fulltraffic_logit50_01_190511_163910"

exp[2300]="alcalahenares_L3h_nomaps_tele60_timeALL_fulltraffic_190515_130939"
exp[2301]="alcalahenares_L3h_mutraff_tele60_uni5x16_timeALL_fulltraffic_logit50_01_190515_133434"

exp[2400]="alcalahenares_2L3h_nomaps_tele60_timeALL_fulltraffic_190515_150939"
exp[2401]="alcalahenares_2L3h_mutraff_tele60_uni5x16_timeALL_fulltraffic_logit50_01_190515_161514"

exp[2410]="alcalahenares_2L3h_nomaps_tele60_timeALL_fulltraffic_190515_182031"
exp[2411]="alcalahenares_2L3h_mutraff_tele60_uni5x8_timeALL_fulltraffic_logit50_01_190515_191357"
exp[2412]="alcalahenares_2L3h_mutraff_tele60_uni5x16_timeALL_fulltraffic_logit50_01_190515_200544"

exp[2420]="alcalahenares_2L3h_nomaps_tele60_timeALL_fulltraffic_190515_202233"
exp[2421]="alcalahenares_2L3h_mutraff_tele60_uni5x8_timeALL_fulltraffic_logit50_01_190515_205714"
exp[2422]="alcalahenares_2L3h_mutraff_tele60_uni5x16_timeALL_fulltraffic_logit50_01_190515_212938"

exp[2430]="alcalahenares_3L3h_nomaps_tele60_timeALL_fulltraffic_190516_084853"
exp[2431]="alcalahenares_3L3h_mutraff_tele60_uni5x8_timeALL_fulltraffic_logit10_01_190521_091537"
exp[2432]="alcalahenares_3L3h_mutraff_tele60_uni5x8_timeALL_fulltraffic_logit20_01_190521_143019"
exp[2433]="alcalahenares_3L3h_mutraff_tele60_uni5x8_timeALL_fulltraffic_logit50_01_190516_091540"
exp[2434]="alcalahenares_3L3h_mutraff_tele60_uni5x8_timeALL_fulltraffic_logit100_01_190521_210550"

exp[2441]="alcalahenares_3L3h_mutraff_tele60_uni5x16_timeALL_fulltraffic_logit10_01_190521_213112"
exp[2442]="alcalahenares_3L3h_mutraff_tele60_uni5x16_timeALL_fulltraffic_logit20_01_190521_220448"
exp[2443]="alcalahenares_3L3h_mutraff_tele60_uni5x16_timeALL_fulltraffic_logit50_01_190516_114311"
exp[2444]="alcalahenares_3L3h_mutraff_tele60_uni5x16_timeALL_fulltraffic_logit100_01_190521_223020"


label[2430]="Alcala 3L - No TWM"
label[2431]="Alcala 3L - TWM[8]  - uniform(5) - 10% usage"
label[2432]="Alcala 3L - TWM[8]  - uniform(5) - 20% usage"
label[2433]="Alcala 3L - TWM[8]  - uniform(5) - 50% usage"
label[2434]="Alcala 3L - TWM[8]  - uniform(5) - 100% usage"
label[2441]="Alcala 3L - TWM[16] - uniform(5) - 10% usage"
label[2442]="Alcala 3L - TWM[16] - uniform(5) - 20% usage"
label[2443]="Alcala 3L - TWM[16] - uniform(5) - 50% usage"
label[2444]="Alcala 3L - TWM[16] - uniform(5) - 100% usage"



# Extra-Small para no saturar y 8 mapas random (500 + 4x200)
exp[20]="XS_grid16_noBastra_reference_fulltraffic_170130_225044"
exp[21]="XS_grid16_Bastra_rand05x8_timeALL_fulltraffic_logit10_161115_092404"
exp[22]="XS_grid16_Bastra_rand05x8_timeALL_fulltraffic_logit50_170130_225435"

exp[23]="XS_grid16_Bastra_rand05x8_timeALL_fulltraffic_logit05_161115_092039"
exp[24]="XS_grid16_Bastra_rand05x8_timeALL_fulltraffic_logit10_161115_092404"
exp[25]="XS_grid16_Bastra_rand05x8_timeALL_fulltraffic_logit20_161115_092738"
exp[26]="XS_grid16_Bastra_rand05x8_timeALL_fulltraffic_logit50_161115_093115"
exp[27]="XS_grid16_Bastra_rand05x8_timeALL_fulltraffic_logit100_161115_093432"

exp[28]="XS_grid16_noBastra_reference_fulltraffic_incident_190603_230928"
exp[29]="XS_grid16_noBastra_reference_fulltraffic_incident_190603_231705"
exp[29]="XS_grid16_noBastra_reference_fulltraffic_incident_190603_233544"

label[20]="Grid16 - XS - no TWM"
label[23]="Grid16 - XS - TWM[8] - normal(5) - 05% usage"
label[25]="Grid16 - XS - TWM[8] - normal(5) - 20% usage"
label[26]="Grid16 - XS - TWM[8] - normal(5) - 50% usage"
label[27]="Grid16 - XS - TWM[8] - normal(5) - 100% usage"
label[28]="Grid16 - XS - no TWM - Single incidents"
label[29]="Grid16 - XS - no TWM - Multiple incidents"<jupyter_output><empty_output><jupyter_text>FORESIGHT AND MAPS<jupyter_code>exp[2599]="alcalahenares_3L3h_nomaps_timeALL_taz5-taz50_01_190605_225531"
exp[2600]="alcalahenares_3L3h_nomaps_timeALL_taz5-taz50_01_190607_163410"
exp[2601]="alcalahenares_3L3h_nomaps_timeALL_taz5-taz50_incident_01_190605_233312"
exp[2602]="alcalahenares_3L3h_nomaps_timeALL_taz5-taz50_foresight_01_190606_072936"
exp[2603]="alcalahenares_3L3h_nomaps_timeALL_taz5-taz50_foresight_incident_01_190606_082447"

label[2599]="No TWM - Alcala - Taz5 to Taz50 - No foresight - Exp1"
label[2600]="No TWM - Alcala - Taz5 to Taz50 - No foresight - Exp2"
label[2601]="No TWM - Alcala - Taz5 to Taz50 - No foresight - INCIDENT"
label[2602]="No TWM - Alcala - Taz5 to Taz50 - Fore2-3-4-10"
label[2603]="No TWM - Alcala - Taz5 to Taz50 - Fore2-3-4-10 - INCIDENT"
exp_nums = [2430,2431,2432,2433,2434]
exp_nums = [2430,2441,2442,2443,2444]
exp_nums = [20,25,26,27]
exp_nums = [20,28,29]
exp_nums = [2600, 2601, 2602, 2603 ]<jupyter_output><empty_output><jupyter_text>Incidents<jupyter_code>exp[2499]="alcalahenares_3L3h_nomaps_tele60_timeALL_fulltraffic_incident_190603_000953"
label[2499]="Alcala 3L - No TWM - Incident in roundabout"

exp_nums = [2430, 2500]

exp[2500]="alcalahenares_3L3h_nomaps_tele60_timeALL_fulltraffic_190604_073604"
exp[2501]="alcalahenares_3L3h_nomaps_tele60_timeALL_fulltraffic_incident_190604_082709"
exp[2502]="alcalahenares_3L3h_mutraff_tele60_uni5x16_timeALL_fulltraffic_logit10_01_190604_091731"

label[2500]="Alcala 3L - no TWM"
label[2501]="Alcala 3L - no TWM - Incident"
label[2502]="Alcala 3L - no TWM 16 MuTraff maps - uniform(5) - 10% usage"

exp_nums = [2500,25]
exps=[]
for n in exp_nums:
    if n in label:
        mylabel=label[n]
    else:
        mylabel=str(n)
    exps.append( { 'exp':exp[n], 'data': load_data(exp[n]), 'label':mylabel})
exps[0]
analyze_experiments( exps )
def load_experiments( exp1, exp2 ):
    data1 = load_data(exp1)
    data2 = load_data(exp2)
    exp_list = [ { "exp_names" : [exp1, exp2] },
                 { "data" : [data1, data2] }
               ]
exp_list = load_experiments( exp[2300], exp[2301] )
analyze_experiments( exp_list )<jupyter_output>Parsing file /Users/alvaro/Desktop/workspace/mutraff/uah-gist-mutraff-bastra/experiments/tmp/alcalahenares_L3h_nomaps_tele60_timeALL_fulltraffic_190515_130939/edge_stats.csv
Parsing file /Users/alvaro/Desktop/workspace/mutraff/uah-gist-mutraff-bastra/experiments/tmp/alcalahenares_L3h_mutraff_tele60_uni5x16_timeALL_fulltraffic_logit50_01_190515_133434/edge_stats.csv<jupyter_text>BACKUP<jupyter_code>ds1 = data[data["traf_halted_veh_num"]>0 ]
measure_name='traf_halted_veh_num'
ds1[ds1[measure_name]>0 ].groupby(['time']).rolling(4).count()
data.groupby(by=['time'])['traf_av_speed','traf_av_occupancy'].mean()<jupyter_output><empty_output><jupyter_text>No se puede sumar el parametro **traf_total_veh_num** porque nos da el valor de los vehiculos totales en el edge en el periodo, pero esto no nos da la metrica de la cantidad de trafico total en el sistema.<jupyter_code>data.head()<jupyter_output><empty_output><jupyter_text>Obtenemos los nodos donde hay vehiculos parados<jupyter_code>data_stopped=data[ (data["traf_halted_veh_num"]>0) & (data['time']<100) ]
data_count_edges_with_halted = data_stopped.groupby(by=['time'])['edge_id'].count()
data_count_edges_with_halted = data_stopped.groupby(by=['time']).count()
data_count_edges_with_halted.unstack()['edge_id']
print("okok")
ds1=data[ (data["traf_halted_veh_num"]>0) & (data['time']<100) ].groupby(by=['time']).sum()
ds1
ds1['traf_total_veh_num']
ds1 = data[data["traf_halted_veh_num"]>0 ].groupby(['time']).sum().unstack()["traf_halted_veh_num"]
print(ds1)
# plot data
fig, axes = plt.subplots(figsize=(15,7))
# use unstack()
ds1.plot(axes=axes)<jupyter_output><empty_output><jupyter_text>Obtenemos los valores agregables en los nodos<jupyter_code>data_aggregated = data.groupby(by=['time'])['traf_waiting_time','traf_halted_veh_num','emission_co2', 'emission_co', 'emission_hc',
       'emission_noise', 'emission_nox', 'emission_PMx', 'consum_epower','consum_fuel'].sum()
data_aggregated
data_mean = data.groupby(by=['time'])['traf_travel_time', 'traf_waiting_time',
       'traf_halted_veh_num', 'traf_av_occupancy',
       'traf_av_speed', 'emission_co2', 'emission_co', 'emission_hc',
       'emission_noise', 'emission_nox', 'emission_PMx', 'consum_epower',
       'consum_fuel'].mean()
data_mean<jupyter_output><empty_output><jupyter_text>GARBAGE AND TESTS<jupyter_code>data = pd.read_csv('ej1.csv')
data
data.groupby(by=['ward', 'division', 'ballot_type', 'office', 'candidate', 'party'])['vote_count'].sum()
print(data)
for d in data:
    print("--> {} = {}".format(d,data[d]))
df2 = pd.DataFrame([data.groupby(by=['ward', 'division', 'ballot_type'])['vote_count'].mean()])
df2

df3=data.groupby(by=['ballot_type'])['vote_count'].mean()
df2 = pd.DataFrame([df3])
df2

df2.unstack()
df3
df3['A']
df3=data.groupby(by=['ballot_type'])['vote_count'].count()
df3<jupyter_output><empty_output><jupyter_text>Draw histograms<jupyter_code>#### Function draw_2histograms
# Plots the comparison between two overlapped histograms based on the datasets.
def draw_histogram( title, label1, experiment1, ds_raw1, label2, experiment2, ds_raw2 ):
    cols = 80
    fig, axes = plt.subplots(figsize=(18,8))

    plt.xlabel('Travel Time')
    plt.ylabel('Frecuency')
    plt.rc('axeses', labelsize=BIG_FONT_SIZE, titlesize=BIG_FONT_SIZE)
    # plt.title("Travel Times Histogram: "+label1+" <--> "+label2 )
    plt.title("Traveltime Histograms Compared" )
    
    # y_vals1, x_bins1, patches1 = axes.hist(travel_times1, histtype='bar', cols, normed=0, facecolor='green', alpha=0.30)
    # y_vals2, x_bins2, patches2 = axes.hist(travel_times2, histtype='bar', cols, normed=0, facecolor='blue', alpha=0.30)
    # [y_vals1, y_vals2], x_bins, patches = axes.hist(travel_times, cols, normed=0, color=['green', 'red'], alpha=0.30, histtype='bar')
    [y_vals1, y_vals2], x_bins, patches = axes.hist(travel_times, cols, normed=0, color=['green', 'red'], histtype='bar', label=['Using TWM','Not using TWM'])
    
    conf_int=0.68
    
    # first histogram ------------------------------------
    x_size1   = len(travel_times1)
    x_mean1   = np.mean(travel_times1)
    x_median1 = np.median(travel_times1)
    x_std1    = np.std(travel_times1)
    x_var1    = np.var(travel_times1)
    x_conf_int1  = st.norm.interval(conf_int, loc=x_mean1, scale=x_std1)
    x_m_conf_int1= st.norm.interval(conf_int, loc=x_mean1, scale=x_std1/np.sqrt(x_size1))
    x_shapiro1=st.shapiro(travel_times1)
    x_shapiro_str1 = confidence_str + ("is not normal" if x_shapiro1[1] < conf_single_stats_confidence else "cannot reject that is normal")
    y_mean1   = np.mean(y_vals1)
    y_median1 = np.median(y_vals1)
    y_std1    = np.std(y_vals1)
    r_mean1   = np.mean(ds1['route_path_num'])
    r_median1 = np.median(ds1['route_path_num'])
    r_std1    = np.std(ds1['route_path_num'])
    r_var1    = np.var(ds1['route_path_num'])
    r_conf_int1= st.norm.interval(conf_int, loc=r_mean1, scale=r_std1)
    r_conf_int1= st.norm.interval(conf_int, loc=r_mean1, scale=r_std1)
    r_m_conf_int1= st.norm.interval(conf_int, loc=r_mean1, scale=r_std1/np.sqrt(x_size1))
    # l_mean1   = np.mean(ds1['route_length'])
    # l_median1 = np.median(ds1['route_length'])
    l_mean1   = 1
    l_median1 = 1

    # add a 'best fit' line
    y1_vals1 = np.append( y_vals1, y_vals1[cols-1] )

    x2_mean1   = [ x_mean1   for i in y1_vals1 ]
    x3_median1 = [ x_median1 for i in y1_vals1 ]
    y2_mean1   = [ y_mean1   for i in x_bins ]
    y3_median1 = [ y_median1 for i in x_bins ]

    # second histogram ------------------------------------
    x_size2   = len(travel_times2)
    x_mean2   = np.mean(travel_times2)
    x_median2 = np.median(travel_times2)
    x_std2    = np.std(travel_times2)
    x_var2    = np.var(travel_times2)
    x_conf_int2  = st.norm.interval(conf_int, loc=x_mean2, scale=x_std2)
    x_m_conf_int2= st.norm.interval(conf_int, loc=x_mean2, scale=x_std2/np.sqrt(x_size2))
    x_shapiro2=st.shapiro(travel_times2)
    x_shapiro_str2 = confidence_str + ("is not normal" if x_shapiro2[1] < conf_single_stats_confidence else "cannot reject that is normal")
    y_mean2   = np.mean(y_vals2)
    y_median2 = np.median(y_vals2)
    y_std2    = np.std(y_vals2)
    r_mean2   = np.mean(ds2['route_path_num'])
    r_median2 = np.median(ds2['route_path_num'])
    r_std2    = np.std(ds2['route_path_num'])
    r_var2    = np.var(ds2['route_path_num'])
    r_conf_int2  = st.norm.interval(conf_int, loc=r_mean2, scale=r_std2)
    r_m_conf_int2= st.norm.interval(conf_int, loc=r_mean2, scale=r_std2/np.sqrt(x_size2))
    r_t_test  = st.ttest_ind( ds1['route_path_num'], ds2['route_path_num'])
    # l_mean2   = np.mean(ds2['route_length'])
    # l_median2 = np.median(ds2['route_length'])
    l_mean2   = 1
    l_median2 = 1
     
    # add a 'best fit' line
    y1_vals2 = np.append( y_vals2, y_vals2[cols-1] )

    x2_mean2   = [ x_mean2   for i in y1_vals2 ]
    x3_median2 = [ x_median2 for i in y1_vals2 ]
    y2_mean2   = [ y_mean2   for i in x_bins ]
    y3_median2 = [ y_median2 for i in x_bins ]

    if( flag_show_demand_stats ):
      theStats = pd.DataFrame([
        [ 'Trips Planned', 'num',  'trips', str_perc(ds_raw1['id'].count(),ds_raw2['id'].count()), ds_raw1['id'].count(), ds_raw2['id'].count()],
        [ 'Trips Routed',  'num',  'trips', str_perc(travel_times1.count(),travel_times2.count()), travel_times1.count(), travel_times2.count()],
        [ 'Trips using TWM',  'num',  'trips', str_perc(travel_times1.count(),len(ds3)), travel_times1.count(), len(ds3)],
        ])
      theStats.columns = [ '', 'Stat', 'Units', 'variation', label1, label2]        
      print_h2(title, "NON-PAIRED STATS - ROUTED DEMAND")
      display(HTML(theStats.to_html()))

    if( flag_show_traveltime_stats ):
      theStats = pd.DataFrame([
        [ 'Travel Time', 'mean',  't.units', str_perc(x_mean1,x_mean2), x_mean1, x_mean2 ],
        [ 'Travel Time', 'median',  't.units', str_perc(x_median1,x_median2), x_median1, x_median2 ],
        # [ 'Travel Time', 'variance',  't.units', str_perc(x_var1,x_var2), x_var1, x_var2 ],
        [ 'Travel Time', 'std.deviation',  't.units', str_perc(x_std1,x_std2), x_std1,x_std2 ],
        [ 'Travel Time', 'Value Confidence Interval',  't.units', '', str(x_conf_int1), str(x_conf_int2) ],
        [ 'Travel Time', 'Mean Confidence Interval',  't.units', '', str(x_m_conf_int1), str(x_m_conf_int2) ],
        [ 'Travel Time', 'Shapiro', '', '', x_shapiro_str1, x_shapiro_str2 ]
        ])
      theStats.columns = [ '', 'Stat', 'Units', 'variation', label1, label2]
      print_h2(title, "NON-PAIRED STATS - TRAVELTIME")
      display(HTML(theStats.to_html()))

    if( flag_show_route_stats ):
      theStats = pd.DataFrame([
        [ 'Route Length', 'mean',  'mts', str_perc(r_mean1,r_mean2), r_mean1, r_mean2 ],
        [ 'Route Length', 'median',  'mts', str_perc(r_median1,r_median2), r_median1, r_median2 ],
        # [ 'Route Length', 'variance',  'mts', str_perc(r_var1,r_var2), r_var1, r_var2 ],
        [ 'Route Length', 'std.deviation',  'mts', str_perc(r_std1,r_std2), r_std1, r_std2 ],
        [ 'Route Length', 'Value Confidence Interval',  'mts', '', str(r_conf_int1), str(r_conf_int2) ],
        [ 'Route Length', 'Mean Confidence Interval',  'mts', '', str(r_m_conf_int1), str(r_m_conf_int2) ],
        ])
      theStats.columns = [ '', 'Stat', 'Units', 'variation', label1, label2]        
      print_h2(title, "NON-PAIRED STATS - ROUTE LENGTH")
      display(HTML(theStats.to_html()))

    print_h2(title, "NON-PAIRED STATS - TRAVELTIMES COMPARED HISTOGRAM")
    if ( flag_print_histogram_shape ):
        line_estimation1 = axes.plot(x_bins, y1_vals1, 'r-', label='Estimation', linewidth=2, color='darkgreen')
        line_estimation2 = axes.plot(x_bins, y1_vals2, 'r-', label='Estimation', linewidth=2, color='darkred')

    maxes_y = np.linspace(0, maxes( maxes(y_vals1), maxes(y_vals2)), num=len(x_bins))
    line_mean1 = axes.plot(x2_mean1, maxes_y, label='Mean', linestyle='--', linewidth=2, color='darkgreen')
    line_median1 = axes.plot(x3_median1, maxes_y, label='Median', linestyle='-', linewidth=2, color='darkgreen')

    line_mean2 = axes.plot(x2_mean2, y1_vals2, label='Mean', linestyle='--', linewidth=2, color='darkred')
    line_median2 = axes.plot(x3_median2, maxes_y, label='Median', linestyle='-', linewidth=2, color='darkred')

    mean_line1   = mlines.Line2D([], [], linestyle='--',color='darkgreen', markersize=1, label='Mean: '+label1)
    median_line1 = mlines.Line2D([], [], linestyle='-', color='darkgreen', markersize=1, label='Median: '+label1)
    mean_line2   = mlines.Line2D([], [], linestyle='--',color='darkred',  markersize=1, label='Mean: '+label2)
    median_line2 = mlines.Line2D([], [], linestyle='-', color='darkred',  markersize=1, label='Median: '+label2)
    red_patch = mpatches.Patch(color='red', label='Using TWM')
    green_patch = mpatches.Patch(color='green', label='Not using TWM')
    axes.legend(handles=[red_patch, green_patch, median_line1, mean_line1, median_line2, mean_line2], loc='upper right', fancybox=True, framealpha=0.5)
    plt.rc('legend', fontsize=MID_FONT_SIZE)
    # plt.axesis([travel_times.min(), travel_times.maxes()+1, 0, 0.03])
    axes.grid(True)

    # If plotting OUTSIDE the notebook, set interactive mode ON/OFF
    # plt.ioff()
    # plt.ion()

    plt.show()
df = pd.DataFrame({
    'name':['john','mary','peter','jeff','bill','lisa','jose'],
    'age':[23,78,22,19,45,33,20],
    'gender':['M','F','M','M','M','F','M'],
    'state':['california','dc','california','dc','california','texas','texas'],
    'num_children':[2,0,0,3,2,1,4],
    'num_pets':[5,1,0,5,2,2,3]
})
df[['name','age','gender','state','num_children','num_pets']]
# plot num_children vs num_pets
df.plot(kind='scatter',x='num_children',y='num_pets',color='red')<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1544299687487317, 'fraction_numerical': 0.0722026056252283, 'mean_word_length': 5.0975006186587475, 'pattern_counts': {'":': 0, '<': 64, '<?xml version=': 0, '>': 69, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '13', 'dir': 'jupyter-structured-clean-dedup', 'id': '473812', 'license': 'Xnet', 'n_tokens_mistral': 11003, 'n_tokens_neox': 10012, 'n_words': 1427, 'path': 'notebooks/TEST_AND_OLDIES/mutraff_global_congestion_statistics-v1.ipynb', 'repo_name': 'uahservtel/uah-gist-mutraff-bastra', 'size': '24639'}