<jupyter_start><jupyter_text>read data  K-means results<jupyter_code>kmean_results_labels = pd.read_csv(
     'results/archieve/kmeans_results_all_TimeZoneCorrected_08202021.csv')
# kmean_results

kmean_results=pd.read_csv('_Data/model data/model_data_TimeZoneCorrected_08192021.csv') #model_data_TimeZoneCorrected_08192021.csv')
# PCA_model_data


kmean_results['KMeans_16_labels']=literal_eval(kmean_results_labels[kmean_results_labels['Cluster_no']
             == 16]['Kmeans_label_list'].values.tolist()[0])

kmean_results.head()
# kmean_results = pd.read_csv(
#     '_Data/Model results/kmean_predictions.csv')


kmean_results['StartTime_decimal'] = kmean_results.apply(
    lambda x: x['StartTime_decimal'] + 24
    if x['StartTime_decimal'] <= 6 else x['StartTime_decimal'],
    axis=1)

#add flag for trips starting or ending at Vanderbilt and Parks
kmean_results['VANDERBILT_trip'] = kmean_results.apply(
    lambda x: 1
    if (x['VANDERBILT_start'] == 1) | (x['VANDERBILT_end'] == 1) else 0,
    axis=1)


kmean_results['PARK_trip'] = kmean_results.apply(
    lambda x: 1
    if (x['PARK_start'] == 1) | (x['PARK_end'] == 1) else 0,
    axis=1)

print(kmean_results.shape)
print(kmean_results.columns)
kmean_results.head()<jupyter_output>(1050497, 50)
Index(['Unnamed: 0', 'trip_id', 'GRID_ID_start', 'GRID_ID_end',
       'enthropy_start', 'MEAN_CBD_start', 'MEAN_URBAN_start', 'MEAN_SU_start',
       'MEAN_RURAL_start', 'MEAN_POP_D_start', 'MEAN_EMP_D_start',
       'MEAN_PAKING_start', 'intersection_density_start', 'enthropy_end',
       'MEAN_CBD_end', 'MEAN_URBAN_end', 'MEAN_SU_end', 'MEAN_RURAL_end',
       'MEAN_POP_D_end', 'MEAN_EMP_D_end', 'MEAN_PAKING_end',
       'intersection_density_end', 'PARK_start', 'VANDERBILT_start',
       'NISSAN_start', 'PARK_end', 'VANDERBILT_end', 'NISSAN_end',
       'tripDistance_miles', 'tripDistance', 'tripDuration',
       'euclidean_distance', 'StartTime_decimal', 'route_directness_rawDist',
       'route_directness_trace', 'average_trip_speed_mph',
       'trip_start_week_no', 'trip_start_hour_no', 'StartTime_AMPeak',
       'StartTime_Day', 'StartTime_PMPeak', 'StartTime_Night', 'weekend_trip',
       'high_trip_count_day', 'DATE_str', 'PRCP', 'TAVG', 'KMeans_16_labels',
   [...]<jupyter_text>Trip data<jupyter_code>trip_data = pd.read_csv(
    '_Data/trip data/start_end_points_cleaned_TimeZoneCorrected_08192021.csv')
trip_data['startDateTime'] = pd.to_datetime(
    trip_data['startDateTime'], utc=True).dt.tz_convert('US/Central')

# trip_data['KMeans_16_labels']=literal_eval(kmean_results_labels[kmean_results_labels['Cluster_no']
#              == 16]['Kmeans_label_list'].values.tolist()[0])
trip_data_merged_label=pd.merge(trip_data,
        kmean_results[['trip_id','KMeans_16_labels']],
        how='left',
        on='trip_id')
trip_data_merged_label.to_csv('_Data/model data/model_results (coords and label)_TimeZoneCorrected_08192021.csv')
trip_data_merged_label.head()
kmeans_results_linkCoords=pd.merge(kmean_results,
         trip_data[['trip_id','startLatitude','startLongitude','endLatitude',
                    'endLongitude','weekend_trip','startDateTime']],
         on='trip_id',
         how='left')
kmeans_results_linkCoords.to_csv('_Data/model data/ClusteringResults_Interpretation_TimeZoneCorrected_08222021_v1.csv')<jupyter_output><empty_output><jupyter_text>Visualizing clustering results<jupyter_code>def plot_graph(cluster_df, selected_cluster, variable_to_plot, title, ax,
               plot_upper_bound, x_label):
    #PURPOSE: Plot kdeplot of a single variable for all cluster type
    #INPUT
    ####ax = axis of gridspec
    ####title: title of the plot
    ####cluster_df: Dataframe with cluster id
    ####selected_cluster: cluster to plot
    ####variable_to_plot: name of the variable to plot
    ####plot_upper_bound: upper bound of the plot
    #OUTPUT: plt plot

    #fig, ax = plt.subplots(figsize=(6,4))

    #run a loop for all the sorted unique values in cluster
    for i in sorted(cluster_df[selected_cluster].unique()):
        #plot kdeplot for each clusters in same plot
        sns.kdeplot(
            cluster_df[cluster_df[selected_cluster] == i][variable_to_plot],
            label=str('C' + str(i)),
            ax=ax)
    ax.set_title(title, fontsize=10)

    ax.legend(fontsize=8, ncol=2)

    ax.axis([0, plot_upper_bound, None, None])
    ax.set_ylabel('Probability', fontsize=9)
    ax.set_xlabel(x_label, fontsize=9)
    ax.tick_params(axis='both', which='major', labelsize=8)
    #return plt


def spyder_plot_for_cluster(cluster_results, cluster_var_col_name,
                            cluster_number, ax, plot_color, var_median,
                            var_mean):

    var_spyder_all = var_median + var_mean
    #slice variable of interest
    data_for_sypder_plot = cluster_results[var_spyder_median + var_spyder_mean]

    #     cluster_var_col_name = str('kmeans_' + str(kmeans_cluster_no))

    #standarize the data
    scaler = preprocessing.MinMaxScaler()
    data_for_sypder_plot_scaled = pd.DataFrame(
        scaler.fit_transform(data_for_sypder_plot), columns=var_spyder_all)
    data_for_sypder_plot_scaled[cluster_var_col_name] = cluster_results[
        cluster_var_col_name]

    #get dataframe of one cluster
    cluster = data_for_sypder_plot_scaled[
        data_for_sypder_plot_scaled[cluster_var_col_name] ==
        cluster_number][var_spyder_all]

    #     #standarize the data
    #     scaler = preprocessing.MinMaxScaler()
    #     cluster = pd.DataFrame(scaler.fit_transform(cluster_0), columns=var_spyder_all)

    labels = np.arange(1,
                       len(np.array(var_spyder_all)) + 1,
                       1)  #np.array(var_spyder_all)

    #concate mean and median columns and sum them to get one column
    stats = pd.concat([
        cluster.loc[:, var_spyder_median].median(axis=0),
        cluster.loc[:, var_spyder_mean].mean(axis=0)
    ],
                      axis=1)
    stats = stats.sum(axis=1)

    angles = np.linspace(0, 2 * np.pi, len(labels), endpoint=False)
    # close the plot
    stats = np.concatenate((stats, [stats[0]]))
    angles = np.concatenate((angles, [angles[0]]))

    fig = plt.figure()
    #     ax = fig.add_subplot(111, polar=True)
    ax.plot(angles,
            stats,
            'o-',
            markersize=2,
            linewidth=1.5,
            color=plot_color,
            label=str('C' + str(cluster_number)))
    ax.fill(angles, stats, alpha=0.05)
    ax.set_thetagrids(angles[:19] * 180 / np.pi, labels)
    ax.set_yticklabels([])
    #     ax.set_title(str('C' + str(cluster_number)))
    ax.grid(True)
#     ax.legend()

var_spyder_median = [
    'tripDistance_miles',
    'tripDuration',
    'route_directness_trace',
    'average_trip_speed_mph',

        'TAVG',
    'MEAN_CBD_start',
    'MEAN_URBAN_start',
    'MEAN_POP_D_start',
    'MEAN_EMP_D_start',
    'MEAN_PAKING_start',
    #     'intersection_density_start',
    'enthropy_start',
    'MEAN_CBD_end',
    'MEAN_URBAN_end',
    'MEAN_POP_D_end',
    'MEAN_EMP_D_end',
    'MEAN_PAKING_end',
    #     'intersection_density_end',
    'enthropy_end'
]

var_spyder_mean = ['high_trip_count_day', 'weekend_trip']

var_spyder_all = var_spyder_median + var_spyder_mean<jupyter_output><empty_output><jupyter_text>Trip start time by clusters  Clusterd by sequence<jupyter_code>selected_cluster = 'KMeans_16_labels'
total_clusters = 16
clusters_per_plot = 3
number_of_plot = 4
plot_split = [0, 3, 7, 11, 15]

style.use('seaborn-darkgrid')
fig = plt.figure(figsize=(4, 9))
#fig.suptitle('K mean cluster 10')

gspec = gridspec.GridSpec(number_of_plot, 1)

#defind colour pallet
custom_palette = sns.color_palette("deep", total_clusters)

for i in np.arange(0, total_clusters, 1):
    if (i <= plot_split[1]):
        sns.kdeplot(kmean_results[kmean_results[selected_cluster] == i]
                    ['StartTime_decimal'],
                    clip=(0, 30),
                    label=str('C' + str(i)),
                    color=custom_palette[i],
                    ax=plt.subplot(gspec[0, 0]))

    elif (i <= plot_split[2]) & (i > plot_split[1]):
        sns.kdeplot(kmean_results[kmean_results[selected_cluster] == i]
                    ['StartTime_decimal'],
                    clip=(0, 30),
                    label=str('C' + str(i)),
                    color=custom_palette[i],
                    ax=plt.subplot(gspec[1, 0]))
    elif (i <= plot_split[3]) & (i > plot_split[2]):
        sns.kdeplot(kmean_results[kmean_results[selected_cluster] == i]
                    ['StartTime_decimal'],
                    clip=(0, 30),
                    label=str('C' + str(i)),
                    color=custom_palette[i],
                    ax=plt.subplot(gspec[2, 0]))
    elif (i <= plot_split[4]) & (i > plot_split[3]):
        sns.kdeplot(kmean_results[kmean_results[selected_cluster] == i]
                    ['StartTime_decimal'],
                    clip=(0, 30),
                    label=str('C' + str(i)),
                    color=custom_palette[i],
                    ax=plt.subplot(gspec[3, 0]))

    plt.xlim(6, 30)
    plt.xticks(np.arange(6, 31, 6),['6 am','12 pm','6 pm','12 pm', '6 am'])
    

#     plt.xticks(np.arange(6, 25, 6))
    plt.yticks(np.arange(0, 0.4, 0.1))
    plt.legend()
plt.tight_layout()
plt.xlabel('Hour of the day')

plt.savefig(str('results/cluster_time_of_day.png'), dpi=900)

# plt.savefig('plots/DailyTrips_Year.png', dpi=900)<jupyter_output><empty_output><jupyter_text>Clustered by group<jupyter_code>selected_cluster = 'KMeans_16_labels'
total_clusters = 16
number_of_plot = 5


#group clusters by id

grouped_cluster_labels=['Group1','Group 2','Group 3','Group 4','Group 5']
dayTime_ShortErrand=[7,10]
utilitarian = [3,9]
evening_Social = [4,8,13]
entertainment_District = [1,5,6,11]
recreational =[0,2,12,14,15]
grouped_clusters = [
    dayTime_ShortErrand, utilitarian, evening_Social, entertainment_District,
    recreational
]


style.use('seaborn-darkgrid')
fig = plt.figure(figsize=(5, 11))
gspec = gridspec.GridSpec(number_of_plot, 1)




grouped_clusters = [
    dayTime_ShortErrand, utilitarian, evening_Social, entertainment_District,
    recreational
]

for grouped_id in np.arange(len(grouped_clusters)):
    for individual_cluster in grouped_clusters[grouped_id]:
        sns.kdeplot(kmean_results[kmean_results[selected_cluster] == individual_cluster]
            ['StartTime_decimal'],
            clip=(0, 30),
            label=str('C' + str(individual_cluster)),
            color=custom_palette[individual_cluster],
            ax=plt.subplot(gspec[grouped_id, 0]))

#     plt.title(grouped_cluster_labels['grouped_id'])
    #add axis labels and ticks
    plt.xlim(6, 30)
    plt.xticks(np.arange(6, 31, 6),['6 am','12 pm','6 pm','12 am', '6 am'])
   
#     plt.xticks(np.arange(6, 25, 6))
    plt.yticks(np.arange(0, 0.4, 0.1), ['','0.1','0.2','0.3'])

    plt.legend()
plt.tight_layout(h_pad=3.0)
plt.xlabel('Hour of the day')

plt.savefig(str('results/cluster_time_of_day_TimeZoneCorrected_grouped_V1.png'),
            bbox_inchs='tight', dpi=900)<jupyter_output><ipython-input-98-525412f03833>:54: MatplotlibDeprecationWarning: savefig() got unexpected keyword argument "bbox_inchs" which is no longer supported as of 3.3 and will become an error two minor releases later
  plt.savefig(str('results/cluster_time_of_day_TimeZoneCorrected_grouped_V1.png'),<jupyter_text>Radar plot for cluster attributes https://www.kaggle.com/typewind/draw-a-radar-chart-with-python-in-a-simple-way/data  clustered by sequence<jupyter_code>fig = plt.figure(figsize=(4, 9))
#fig.suptitle('K mean cluster 10')

gspec = gridspec.GridSpec(4, 1)
ax0 = plt.subplot(gspec[0, 0], polar=True)
ax1 = plt.subplot(gspec[1, 0], polar=True)
ax2 = plt.subplot(gspec[2, 0], polar=True)
ax3 = plt.subplot(gspec[3, 0], polar=True)

#defind colour pallet
custom_palette = sns.color_palette("deep", total_clusters)

for i in np.arange(0, total_clusters, 1):
    if (i <= plot_split[1]):
        spyder_plot_for_cluster(kmean_results,
                        selected_cluster,
                        i,
                        ax0,
                        custom_palette[i],
                        var_spyder_median,
                        var_spyder_mean)
    elif (i <= plot_split[2]) & (i > plot_split[1]):
        spyder_plot_for_cluster(kmean_results,
                selected_cluster,
                i,
                ax1,
                custom_palette[i],
                var_spyder_median,
                var_spyder_mean)
    elif (i <= plot_split[3]) & (i > plot_split[2]):
        spyder_plot_for_cluster(kmean_results,
                selected_cluster,
                i,
                ax2,
                custom_palette[i],
                var_spyder_median,
                var_spyder_mean)        
    elif (i <= plot_split[4]) & (i > plot_split[3]): 
        spyder_plot_for_cluster(kmean_results,
                selected_cluster,
                i,
                ax3,
                custom_palette[i],
                var_spyder_median,
                var_spyder_mean)        
fig.tight_layout()       

fig.savefig(str('results/cluster_attributes_TimeZoneCorrected.png'),dpi=900)<jupyter_output><empty_output><jupyter_text>Clustered by group<jupyter_code>var_spyder_median = [
    'tripDistance_miles',
    'tripDuration',
    'route_directness_trace',
    'average_trip_speed_mph'

#         'TAVG',
#     'MEAN_CBD_start',
#     'MEAN_URBAN_start',
#     'MEAN_POP_D_start',
#     'MEAN_EMP_D_start',
#     'MEAN_PAKING_start',
#     #     'intersection_density_start',
#     'enthropy_start',
#     'MEAN_CBD_end',
#     'MEAN_URBAN_end',
#     'MEAN_POP_D_end',
#     'MEAN_EMP_D_end',
#     'MEAN_PAKING_end',
#     #     'intersection_density_end',
#     'enthropy_end'
]

var_spyder_mean = ['high_trip_count_day', 'weekend_trip']

var_spyder_all = var_spyder_median + var_spyder_mean


style.use('seaborn-darkgrid')
fig = plt.figure(figsize=(4, 9))

#defind colour pallet
custom_palette = sns.color_palette("deep", total_clusters)



gspec = gridspec.GridSpec(5, 1)
ax0 = plt.subplot(gspec[0, 0], polar=True)
ax1 = plt.subplot(gspec[1, 0], polar=True)
ax2 = plt.subplot(gspec[2, 0], polar=True)
ax3 = plt.subplot(gspec[3, 0], polar=True)
ax4 = plt.subplot(gspec[4, 0], polar=True)

ax=[ax0, ax1, ax2, ax3, ax4]


custom_palette = sns.color_palette("deep", total_clusters)

for grouped_id in np.arange(len(grouped_clusters)):
    for individual_cluster in grouped_clusters[grouped_id]:
        #defind colour pallet
        spyder_plot_for_cluster(kmean_results,
                        selected_cluster,
                        individual_cluster,
                        ax[grouped_id],
                        custom_palette[individual_cluster],
                        var_spyder_median,
                        var_spyder_mean)

fig.tight_layout(pad=3)

fig.savefig(str('results/cluster_attributes_TimeZoneCorrected-grouped_V1.png'), dpi=900)

# plt.savefig('plots/DailyTrips_Year.png', dpi=900)<jupyter_output><empty_output><jupyter_text>Radar chart by variable<jupyter_code>plot_var_spyder=[    'tripDistance_miles',
    'tripDuration',
    'route_directness_trace',
    'average_trip_speed_mph']
plot_var_spyder_labels=[*plot_var_spyder, plot_var_spyder[0]]


kmean_results_df=kmean_results[plot_var_spyder+['KMeans_16_labels']].groupby(
    'KMeans_16_labels').mean().T#.reset_index()
kmean_results_df

#var name
var_name='average_trip_speed_mph'

def plot_spider_byVar(var_name,)

labels=[str('C'+str(col)) for col in kmean_results_df.columns]
stats=kmean_results_df.loc[var_name,].values

angles=np.linspace(0, 2*np.pi, len(labels), endpoint=False)
# close the plot
stats=np.concatenate((stats,[stats[0]]))
angles=np.concatenate((angles,[angles[0]]))
labels=np.concatenate((labels,[labels[0]]))
print(len(stats))
print(len(angles))
print(len(labels))
# labels


fig=plt.figure()
ax = fig.add_subplot(111, polar=True)
ax.plot(angles, stats, 'o-', linewidth=2)
ax.fill(angles, stats, alpha=0.25)
ax.set_thetagrids(angles * 180/np.pi, labels)
ax.set_title('a')
ax.grid(True)

rename_variables = {
    'tripDistance_miles': 'Trip distance (miles)',
    'tripDuration': 'Trip duration (minutes)',
    'route_directness_trace': 'Route directness ratio',
    'average_trip_speed_mph': 'Average trip speed (mph)',
    'high_trip_count_day': 'Trips on special event',
    'weekend_trip': 'Weekend trips',
    'StartTime_AMPeak': 'Trip during 7 am and 10 am',
    'StartTime_Day': 'Trip during 10 am and 4 pm',
    'StartTime_PMPeak': 'Trip during 4 pm and 8 pm',
    'StartTime_Night': 'Trip during 8 pm and 7 am',
    'PRCP': 'Average daily precipitation',
    'TAVG': 'Average daily temperature',
    'MEAN_CBD_start': 'Proportion of CBD land use at origin',
    'MEAN_URBAN_start': 'Proportion of urban land use at origin',
    'MEAN_SU_start': 'Proportion of sub-urban land use at origin',
    'MEAN_RURAL_start': 'Proportion of rural land use at origin',
    'MEAN_POP_D_start': 'Average population density at origin',
    'MEAN_EMP_D_start': 'Average employment density at origin',
    'MEAN_PAKING_start': 'Average parking density at origin',
    'intersection_density_start':
    'Intersection density (per sq. miles) at origin',
    'enthropy_start': 'Entropy at origin',
    'MEAN_CBD_end': 'Proportion of CBD land use at destination',
    'MEAN_URBAN_end': 'Proportion of urban land use at destination',
    'MEAN_SU_end': 'Proportion of sub-urban land use at destination',
    'MEAN_RURAL_end': 'Proportion of rural land use at destination',
    'MEAN_POP_D_end': 'Average population density at destination',
    'MEAN_EMP_D_end': 'Average employment density at destination',
    'MEAN_PAKING_end': 'Average parking density at destination',
    'intersection_density_end':
    'Intersection density (per sq. miles) at destination',
    'enthropy_end': 'Entropy at destination'
}

for x,y in zip( np.arange(1,len(np.array(var_spyder_all))+1,1),np.array(var_spyder_all)):
    print('{0}: {1}'.format(x,rename_variables[y]))<jupyter_output>1: Trip distance (miles)
2: Trip duration (minutes)
3: Route directness ratio
4: Average trip speed (mph)
5: Average daily temperature
6: Proportion of CBD land use at origin
7: Proportion of urban land use at origin
8: Average population density at origin
9: Average employment density at origin
10: Average parking density at origin
11: Entropy at origin
12: Proportion of CBD land use at destination
13: Proportion of urban land use at destination
14: Average population density at destination
15: Average employment density at destination
16: Average parking density at destination
17: Entropy at destination
18: Trips on special event
19: Weekend trips<jupyter_text>Redefine clusters https://www.youtube.com/watch?v=T2mQiesnx8s<jupyter_code>day_errand = [5, 8, 14]  #1
utilitarian_trips = [1, 2, 3]  #2
social = [6, 9, 13]  #3
bar_hopping = [4, 7, 10]  #4
social_recreational = [0, 11, 12, 15]  #5

reclassify_cluster_label=['Daytime short errand', 'Utilitarian', 'Evening social',
                         'Night-time Entertainment District', 'Recreational']


def reclassify_cluster(cluster_value_for_reclassify):
    if cluster_value_for_reclassify in day_errand:
        return 0
    if cluster_value_for_reclassify in utilitarian_trips:
        return 1
    if cluster_value_for_reclassify in social:
        return 2
    if cluster_value_for_reclassify in bar_hopping:
        return 3
    if cluster_value_for_reclassify in social_recreational:
        return 4




kmean_results['reclassified_cluster'] = kmean_results.apply(
    lambda x: reclassify_cluster(x['kmeans_16']), axis=1)

kmean_results.head()<jupyter_output><empty_output><jupyter_text>Regrouped cluster results in table<jupyter_code>var_spyder_all = var_spyder_median + var_spyder_mean

cluster_number = 16
selected_cluster_to_group = ['KMeans_16_labels']#['reclassified_cluster']



#descriptive stats
summary_median = kmean_results[var_spyder_median +
                               selected_cluster_to_group].groupby(selected_cluster_to_group).agg(
                                   [np.mean])
summary_mean = kmean_results[var_spyder_mean +
                             selected_cluster_to_group].groupby(selected_cluster_to_group).agg(
                                 [np.mean])

summary = pd.concat([summary_median, summary_mean], axis=1)


total_special_trips=kmean_results.groupby(selected_cluster_to_group[0])['high_trip_count_day'].sum().sum()
special_trips_by_cluster=kmean_results.groupby(selected_cluster_to_group[0])['high_trip_count_day'].sum()
summary['proportion_of_special_day_trip'] = special_trips_by_cluster/total_special_trips*100

vanderbilt_trips=kmean_results.groupby(selected_cluster_to_group[0])['VANDERBILT_trip'].sum().sum()
vanderbilt_by_cluster=kmean_results.groupby(selected_cluster_to_group[0])['VANDERBILT_trip'].sum()
summary['proportion_of_vanderbilt_trip'] = vanderbilt_by_cluster/vanderbilt_trips*100

park_trips=kmean_results.groupby(selected_cluster_to_group[0])['PARK_trip'].sum().sum()
park_by_cluster=kmean_results.groupby(selected_cluster_to_group[0])['PARK_trip'].sum()
summary['proportion_of_park_trip'] = park_by_cluster/park_trips*100

#add proportion column
summary['proportion_of_trips'] = kmean_results.groupby(
    selected_cluster_to_group[0]).size() / len(kmean_results) * 100
# summary['proportion_of_trips'] = kmean_results.groupby(
#     selected_cluster_to_group[0]).size() / len(kmean_results) * 100

total_VMT=kmean_results.groupby(selected_cluster_to_group[0])['tripDistance_miles'].sum().sum()
total_VMT_by_cluster=kmean_results.groupby(selected_cluster_to_group[0])['tripDistance_miles'].sum()
summary['proportion_by_VMT'] = total_VMT_by_cluster/total_VMT*100

total_duration=kmean_results.groupby(selected_cluster_to_group[0])['tripDuration'].sum().sum()
total_duration_by_cluster=kmean_results.groupby(selected_cluster_to_group[0])['tripDuration'].sum()
summary['proportion_by_duration'] = total_duration_by_cluster/total_duration*100



summary=summary.transpose().rename(index=rename_variables)

summary.to_csv(
    str('results/regrouped_results_TimeZoneCorrected_V1_08222021.csv'))

summary
summary_time_day=kmean_results[[ 'StartTime_AMPeak',
       'StartTime_Day', 'StartTime_PMPeak', 
               'StartTime_Night','KMeans_16_labels']].groupby(
    ['KMeans_16_labels']).agg(['sum']) #'weekend_trip'
summary_time_day
style.use('seaborn-darkgrid')

summary_time_day_prop=summary_time_day.apply(lambda x :100 * x / float(x.sum()), axis=1)

fig, ax = plt.subplots(figsize=(8, 9))
p = sns.heatmap(summary_time_day_prop,cmap='RdBu', ax=ax)  #RdBu

plt.tight_layout()<jupyter_output><empty_output><jupyter_text>Temporal distrubution<jupyter_code>selected_cluster = 'kmeans_16'
rolling_avg_data = pd.merge(kmean_results[['trip_id', selected_cluster]],
                          trip_data[['trip_id', 'startDateTime']],
                          on='trip_id',
                          how='inner').set_index('startDateTime')


rolling_avg_data['reclassified_cluster'] = rolling_avg_data.apply(
    lambda x: reclassify_cluster(x[selected_cluster]), axis=1)


rolling_avg_data1=rolling_avg_data.reset_index()






frame_reclassify = {}
for i in np.arange(0, int(len(reclassify_cluster_label)), 1):
#     print(i)
    frame_reclassify.update({
        str(reclassify_cluster_label[i]):
        rolling_avg_data[rolling_avg_data['reclassified_cluster'] ==
                         i].index.value_counts().resample('D').sum()
    })
    
rolling_avg_df = pd.DataFrame(frame_reclassify,index=pd.date_range('2018-9-1', periods=365, freq='D'))
rolling_avg_df = rolling_avg_df.fillna(0)
rolling_avg_df.head()<jupyter_output><empty_output><jupyter_text>Time of the year  by numbers<jupyter_code>style.use('seaborn-darkgrid')
sns.set_palette("deep")
from matplotlib.dates import DateFormatter
# custom_palette=sns.color_palette("deep", 5)
fig, ax = plt.subplots(figsize=(10, 5))
for i in rolling_avg_df.columns:
    plt.plot(rolling_avg_df[i].rolling(21,center=True).mean(), label=i)

date_form = DateFormatter("%b %Y")
ax.xaxis.set_major_formatter(date_form)

#axis label
plt.xlabel('Time of the year', fontsize=13)
plt.ylabel('Average daily trips', fontsize=13)

_ = ax.tick_params(axis='both', which='major', labelsize=11)
_ = ax.tick_params(axis='both', which='minor', labelsize=11)
plt.legend(fontsize=11)
plt.tight_layout()
plt.savefig('results/TemporalDistribution_RegroupedCluster_TimeOfYear.png', dpi=300)
plt.legend()<jupyter_output><empty_output><jupyter_text>by percentage<jupyter_code>total_trips_by_grouped_cluster=kmean_results.groupby(
    selected_cluster_to_group[0]).size()

rolling_avg_by_percent=rolling_avg_df.div(list(total_trips_by_grouped_cluster))*100
style.use('seaborn-darkgrid')
sns.set_palette("deep")
from matplotlib.dates import DateFormatter
# custom_palette=sns.color_palette("deep", 5)
fig, ax = plt.subplots(figsize=(10, 5))
for i in rolling_avg_df.columns:
    plt.plot(rolling_avg_by_percent[i].rolling(21, center=True).mean(),
             label=i)

date_form = DateFormatter("%b %Y")
ax.xaxis.set_major_formatter(date_form)

#axis label
plt.xlabel('Time of the year', fontsize=13)
plt.ylabel('Percentage of usage-grouped trips per day', fontsize=13)

_ = ax.tick_params(axis='both', which='major', labelsize=11)
_ = ax.tick_params(axis='both', which='minor', labelsize=11)
plt.legend(fontsize=11)
plt.tight_layout()
plt.savefig('results/TemporalDistribution_RegroupedCluster_TimeOfYearV2.png',
            dpi=300)
plt.legend()<jupyter_output><empty_output><jupyter_text>Day of the week<jupyter_code># week_no_value=6
def relabeled_weekno(week_no_value):
    if week_no_value<4:
        return 0
    elif week_no_value==4:
        return 1
    else:
        return week_no_value-3
    

index_array = [np.arange(0, 4, 1), np.arange(0, 24, 1)]
cluster_time_summary = pd.DataFrame(index=pd.MultiIndex.from_product(
    index_array, names=('week_day', 'hour_no')))

data_all_trips_reidentified = rolling_avg_data1.reset_index()
# kmean_results['reclassified_cluster'].unique()
cluster_time_summary
for i in np.arange(0, int(len(reclassify_cluster_label)), 1):
    plot_data = data_all_trips_reidentified[
        data_all_trips_reidentified['reclassified_cluster'] == i]

    plot_data_count = pd.DataFrame(
        plot_data['startDateTime'].value_counts().resample(
            'H').count()).rename(columns={
                'startDateTime': reclassify_cluster_label[i]
            }).reset_index()

    # #creating column for weekday no

    plot_data_count['week_day'] = plot_data_count.apply(
        lambda x: relabeled_weekno(x['index'].weekday()), axis=1)
    plot_data_count['hour_no'] = plot_data_count.apply(
        lambda x: x['index'].hour, axis=1)

    plot_data_summary = plot_data_count.groupby(
        ['week_day', 'hour_no'])[reclassify_cluster_label[i]].mean()

    cluster_time_summary = pd.merge(cluster_time_summary,
                                    plot_data_summary,
                                    on=['week_day', 'hour_no'],
                                    how='left')
# cluster_time_summary = cluster_time_summary.fillna(0)
cluster_time_summary
#creating column for weekday no
# data['week_no'] = data.apply(lambda x: x['startDateTime'].weekday(), axis=1)
# data['hour_no'] = data.apply(lambda x: x['startDateTime'].hour, axis=1)
style.use('seaborn-darkgrid')
sns.set_palette("deep")

fig, ax = plt.subplots(figsize=(11, 4))

# for i in np.arange(0, total_clusters, 1):
#     plot_data = kmean_results[kmean_results[selected_cluster] == i]

#     plot_data.groupby(['trip_start_week_no', 'trip_start_hour_no'
#                        ]).size().plot(ax=ax)

cluster_time_summary.plot(ax=ax)

for x in np.arange(0, 24 * 5, 24):
    ax.axvline(x=x, color='red', linewidth=0.9, linestyle='--')

day_label = ['Monday-Thursday', 'Friday', 'Saturday', 'Sunday']
day_label_position=[6,33.8,57.1,81.5]

for x in np.arange(1, 5, 1):
    ax.text(day_label_position[x - 1],#-20 + 0.5 * len(day_label[x - 1]) + 24 * x,
            -50,
            day_label[x - 1],
            color='black',
            fontsize=11)

# axis tick label
# ax.set_yticklabels['12,24']

# ax.xaxis.set_minor_locator(MultipleLocator(6))
# ax.xaxis.set_major_locator(MultipleLocator(12))
# ax.set_xticklabels([0, 0, 12, 24, 12, 24, 12, 24, 12, 24, 24, 24, 24])

plt.xticks(np.arange(0,24*4+6,6),['']+['6 am','12 pm','6 pm','12 am']*4)

#axis label
plt.xlabel('Time of the week', fontsize=13, labelpad=15)
plt.ylabel('Average trip volume per hour', fontsize=13)

_ = ax.tick_params(axis='both', which='major', labelsize=11)
_ = ax.tick_params(axis='both', which='minor', labelsize=11)
plt.legend(fontsize=11,loc='upper left', bbox_to_anchor=(0.04, 1))
plt.tight_layout()
plt.savefig('results/TemporalDistribution_RegroupedCluster_TimeOfWeek_V2.png', dpi=300)<jupyter_output><empty_output><jupyter_text>Spatial Distribution  All trips<jupyter_code>all_cluster_start = trip_data[['trip_id', 'startLatitude', 'startLongitude']]

# all_cluster_end = trip_data[['trip_id', 'endLatitude', 'endLongitude']]

# read last config
with open(str('_Data/kepler config/config_hist_all.json'), 'r') as fp:
    config_1 = json.load(fp)


map_start = keplergl.KeplerGl(height=600,
                              data={'data': all_cluster_start},
                              config=config_1)
# this will save map with provided data and config
map_start.save_to_html(file_name=str('results/spatial plots/all_start_heatmap.html'),
                       read_only=False)


#end points
all_cluster_end = trip_data[['trip_id', 'endLatitude', 'endLongitude']]
rep = {
    "data_start": 'data_end',
    "startLatitude": "endLatitude",
    "startLongitude": "endLongitude",
    "start": "end"
}


def replace_all(text, dic):
    for i, j in dic.items():
        text = text.replace(i, j)
    return text


config_end = json.loads(replace_all(json.dumps(config_1), rep))

map_end = keplergl.KeplerGl(height=600,
                            data={'data': all_cluster_end},
                            config=config_end)
# this will save map with provided data and config
map_end.save_to_html(file_name=str('results/spatial plots/all_end_heatmap.html'),
                     read_only=False)<jupyter_output><empty_output><jupyter_text>Clusters<jupyter_code>for cluster_no in np.arange(0, int(len(reclassify_cluster_label)), 1):
#     cluster_no=0
    #slice trips in cluster
    reclass_cluster = rolling_avg_data[rolling_avg_data.reclassified_cluster ==
                                       cluster_no]

    #merge with start and end points
    reclass_cluster_start = pd.merge(
        reclass_cluster, trip_data, on='trip_id',
        how='left')[['trip_id', 'startLatitude', 'startLongitude']]

    reclass_cluster_end = pd.merge(reclass_cluster,
                                   trip_data,
                                   on='trip_id',
                                   how='left')[[
                                       'trip_id', 'endLatitude', 'endLongitude'
                                   ]]

    plot_data_name = str('Reclassfied_cluster' + str(cluster_no))

    # read last config
    with open(str('_Data/kepler config/config_hist.json'), 'r') as fp:
        config_1 = json.load(fp)
        
    config_start = json.loads(
        json.dumps(config_1).replace('Reclassfied_cluster0', plot_data_name))

    #for end config
    rep = {
        "Reclassfied_cluster0": plot_data_name,
        "startLatitude": "endLatitude",
        "startLongitude": "endLongitude",
        "start": "end"
    }

    def replace_all(text, dic):
        for i, j in dic.items():
            text = text.replace(i, j)
        return text

    config_end = json.loads(replace_all(json.dumps(config_1), rep))

    map_start = keplergl.KeplerGl(height=600,
                                  data={plot_data_name: reclass_cluster_start},
                                  config=config_start)
    # this will save map with provided data and config
    map_start.save_to_html(file_name=str('results/spatial plots/Regrouped_C'+str(cluster_no) + '_start_heatmap.html'),
                           read_only=False)

    map_end = keplergl.KeplerGl(height=600,
                                data={plot_data_name: reclass_cluster_end},
                                config=config_end)
    # this will save map with provided data and config
    map_end.save_to_html(file_name=str('results/spatial plots/Regrouped_C'+str(cluster_no) + '_end_heatmap.html'),
                         read_only=False)
    map_start
map_start
# ##SAVE CONFIG

# config_hist = map_start.config

# #save the current config
# with open(
#         '_Data/kepler config/config_hist.json',
#         'w') as fp:
#     json.dump(config_hist, fp)<jupyter_output><empty_output><jupyter_text>OD matrix<jupyter_code>for cluster_no in np.arange(0, int(len(reclassify_cluster_label)), 1):
    # cluster_no=3
    #slice trips in cluster
    reclass_cluster = rolling_avg_data[rolling_avg_data.reclassified_cluster ==
                                       cluster_no]

    #merge with start and end points
    reclass_cluster = pd.merge(reclass_cluster,
                               trip_data,
                               on='trip_id',
                               how='left')[[
                                   'trip_id', 'startLatitude',
                                   'startLongitude', 'endLatitude',
                                   'endLongitude'
                               ]].sample(1000)

    plot_data_name = str('Reclassfied_cluster' + str(cluster_no))

    # read last config
    with open('config_OD.json', 'r') as fp:
        config_OD = json.load(fp)
    # config_OD = json.loads(json.dumps(config_1).replace('Reclassfied_cluster0',plot_data_name))

    #for end config
    rep = {
        "Reclassfied_cluster0": plot_data_name,
    }

    #     def replace_all(text, dic):
    #         for i, j in dic.items():
    #             text = text.replace(i, j)
    #         return text

    config_OD_edit = json.loads(replace_all(json.dumps(config_OD), rep))

    map_OD = keplergl.KeplerGl(height=600,
                               data={plot_data_name: reclass_cluster},
                               config=config_OD_edit)
    # this will save map with provided data and config
    map_OD.save_to_html(file_name=str('results/spatial plots/OD_' +
                                      str(cluster_no) + '.html'),
                        read_only=False)
# map_start.config

# ##SAVE CONFIG

# config_OD = map_start.config

# #save the current config
# with open(
#         '../1_Data/Model results/kmeans/aggressive_cleaning/Visualization/reclassified_kepler/config_OD.json',
#         'w') as fp:
#     json.dump(config_OD, fp)<jupyter_output><empty_output><jupyter_text>Data for Chord diagram<jupyter_code>#slice data
kmeans_results_export = kmean_results[['trip_id', 'reclassified_cluster']]

#label dictionary
regrouped_label_dict = {
    0: 'Daytime short errand',
    1: 'Utilitarian',
    2: 'Evening social',
    3: 'Night-time Entertainment District',
    4: 'Recreational'
}

kmeans_results_export['regrouped_cluster_label'] = kmeans_results_export[
    'reclassified_cluster'].map(regrouped_label_dict)

kmeans_results_export=kmeans_results_export.drop(columns=['reclassified_cluster'])
kmeans_results_export.head()
# Landuse classification by grid
grid_clf=gpd.read_file('_Data/Landuse data/Shapefiles/LandUse_Classification.shp')
grid_clf_byGrid=grid_clf[['Fishnet_25','Landuse']]
grid_clf_byGrid.head()
#grid id for trip origin and destination
trip_grid = gpd.read_file('_Data/Landuse data/Shapefiles/Start_end_grid_AllSegment.shp')
trip_grid=trip_grid[['trip_id', 'GRID_ID','GRID_ID_1']].rename(columns={'GRID_ID':'Origin_GRID',
                                                             'GRID_ID_1':'Destination_GRID'})
#merge
chord_diagram_df = pd.merge(kmeans_results_export,
                            trip_grid,
                            on='trip_id',
                            how='left')
chord_diagram_df['Origin_Landuse'] = chord_diagram_df['Origin_GRID'].map(
    dict(dict(grid_clf_byGrid.values)))
chord_diagram_df['Distination_Landuse'] = chord_diagram_df['Destination_GRID'].map(
    dict(dict(grid_clf_byGrid.values)))
chord_diagram_df
crosstab_Landuse=pd.crosstab(chord_diagram_df['Origin_Landuse'],
                            [chord_diagram_df['Distination_Landuse']],
                            margins=False)
crosstab_Landuse.to_csv('_Data/Chord diagram/crosstab_landuse.csv')
crosstab_Landuse
crosstab_Landuse_withUseType=pd.crosstab(chord_diagram_df['Origin_Landuse'],
                            [chord_diagram_df['Distination_Landuse'],chord_diagram_df['regrouped_cluster_label']],
                            margins=False)
crosstab_Landuse_withUseType.to_csv('_Data/Chord diagram/crosstab_landuse_withUseType.csv')
crosstab_Landuse_withUseType<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.12400948735917201, 'fraction_numerical': 0.02007978006576465, 'mean_word_length': 3.6512473360912625, 'pattern_counts': {'":': 9, '<': 81, '<?xml version=': 0, '>': 77, 'https://': 2, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 23, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '18', 'dir': 'jupyter-structured-clean-dedup', 'id': '175977', 'license': 'MIT', 'n_tokens_mistral': 13762, 'n_tokens_neox': 12908, 'n_words': 2079, 'path': '4_Cluster interpretation.ipynb', 'repo_name': 'niteshshah12/scooter-trip-pattern-paper', 'size': '37102'}