<jupyter_start><jupyter_text>Imports and config<jupyter_code># Put these at the top of every notebook, to get automatic reloading and inline plotting
%reload_ext autoreload
%autoreload 2
%matplotlib inline
import pandas as pd
import numpy as np
import partridge as ptg
import datetime
import os
import matplotlib.pyplot as plt

#import gtfs_utils as gu
#import gtfstk

import altair as alt
alt.renderers.enable('notebook')<jupyter_output><empty_output><jupyter_text>Read files<jupyter_code>import glob
rs = (pd.concat([pd.read_pickle(f, compression='gzip') 
                 for f in glob.glob('data\\gtfs_stats_MOD_ffill\\2018-*_route_stats.pkl.gz')])
      .assign(date = lambda x: pd.to_datetime(x.date))
      .assign(dayofweek = lambda x: x['date'].dt.strftime('%A')))
rs.shape
rs.head().T
total_trips = (rs.set_index('date').resample('D').num_trips.sum().reset_index().groupby(pd.Grouper(key='date', freq='W-SUN')).agg(['min', 'mean', 'max', 'sum']).reset_index())
total_trips.columns = ['date', 'min', 'mean', 'max', 'sum']
total_trips.head()
weekday_total = (rs.set_index('date').resample('D').num_trips.sum().reset_index()
                 .assign(dayofweek = lambda x: x.date.dt.strftime('%A'))
                 .set_index('date'))

weekday_total =  (weekday_total[~weekday_total.dayofweek.isin(['Friday', 'Saturday'])]
                  .groupby(pd.Grouper(level=0, freq='W-SUN'))
                  .agg(['min', 'mean', 'max', 'idxmax', 'idxmin'])
                  .reset_index())
weekday_total.columns = ['date', 'weekday_min', 'weekday_mean', 'weekday_max', 'max_weekday', 'min_weekday']
alltots = total_trips.merge(weekday_total)
points = alt.Chart(total_trips).mark_point(filled=True).encode(
    alt.Y(
        'mean',
        scale=alt.Scale(zero=False),
        axis=alt.Axis(title='Daily Trips')
    ),
    x='date:T',
    color=alt.value('black')
)

error_bars = alt.Chart(total_trips).mark_rule().encode(
    x='date:T',
    y='min',
    y2='max'
)

(points + error_bars).properties(height=600, width=800)
weekday_min_color = alt.condition("datum.weekday_min != datum.min",
                                 alt.value("#06982d"),
                                 alt.value("#ae1325"))

rule = alt.Chart(alltots).mark_rule().encode(
    alt.X(
        'date:T',
    ),
    alt.Y(
        'min',
        scale=alt.Scale(zero=False),
        axis=alt.Axis(title='Daily Trips')
    ),
    alt.Y2('max'),
    color=alt.value("#ae1325"),
    tooltip=[alt.Tooltip('date:T', format='%A, %B %e'),'min', 'max', 'mean']
)

bar = alt.Chart(alltots).mark_bar().encode(
    x='date:T',
    y='weekday_min',
    y2='weekday_max',
    color=alt.value("#ae1325"),
    tooltip=[alt.Tooltip('date:T', format='%A, %B %e'), 'weekday_mean', alt.Tooltip('min_weekday:T', format='%A, %B %e'), alt.Tooltip('max_weekday:T', format='%A, %B %e')]
)

(rule + bar + points).properties(height=600, width=800)
wd_avg = rs.groupby('dayofweek').num_trips.sum() / rs.groupby('dayofweek').date.nunique() 
wd_avg
eid_routes = rs[(rs.date>'2018-08-19') & (rs.date<'2018-08-25')]
week_before_routes = rs[(rs.date<='2018-08-17') & (rs.date>'2018-08-12')]
id_num_trips = eid_routes.groupby('dayofweek').num_trips.sum()
id_num_trips 
week_before = week_before_routes.groupby('dayofweek').num_trips.sum()
comp = pd.concat([wd_avg, week_before, id_num_trips], axis=1, keys = ['avg', 'week_before', 'eid']).reindex(['Monday', 'Tuesday', 'Wednesday', 'Thursday', 'Friday'])
comp[['week_before', 'eid']].plot(figsize=(15,10), kind='bar')
sums = comp.loc[['Tuesday', 'Wednesday', 'Thursday']].sum()
sums['eid'] / sums['week_before']
comp['pct_change'] = (100*(1-np.round((comp.week_before/comp.eid), 2))).astype(int)
comp
id_num_trips_zone = eid_routes.groupby(['dayofweek', 'start_zone']).num_trips.sum()
week_before_zone = week_before_routes.groupby(['dayofweek', 'start_zone']).num_trips.sum()

comp_zone = (pd.concat([week_before_zone, id_num_trips_zone], axis=1, keys = ['week_before', 'eid'])
             #.stack()
             .reset_index()
             #.rename(columns={'level_2': 'week', 0: 'num_trips'})
            )


comp_zone['pct_change'] = (-100*(1-np.round((comp_zone.eid/comp_zone.week_before), 2))).astype(int)

sorter = ['Monday', 'Tuesday', 'Wednesday', 'Thursday', 'Friday']
comp_zone.dayofweek = comp_zone.dayofweek.astype("category")
comp_zone.dayofweek.cat.set_categories(sorter, inplace=True)
comp_zone=comp_zone.sort_values('dayofweek')
comp_zone
alt.Chart(comp_zone[~comp_zone.dayofweek.isin(('Saturday', 'Sunday', 'Monday'))]).mark_point().encode(
    x = alt.X('dayofweek:N', ),
    y = 'pct_change',
    color = 'start_zone:N',
    tooltip = 'start_zone',
).properties(height=600, width=800)
comp_zone.sort_values('pct_change').to_csv('data/eid_comp_zone.csv')
id_num_trips_zone_inside = eid_routes[eid_routes.start_zone==eid_routes.end_zone].groupby(['dayofweek', 'start_zone', 'agency_name']).num_trips.sum()
week_before_zone_inside = week_before_routes[week_before_routes.start_zone==week_before_routes.end_zone].groupby(['dayofweek', 'start_zone', 'agency_name']).num_trips.sum()

comp_zone_inside = (pd.concat([week_before_zone_inside, id_num_trips_zone_inside], axis=1, keys = ['week_before', 'eid'])
             #.stack()
             .reset_index()
             #.rename(columns={'level_2': 'week', 0: 'num_trips'})
            )


comp_zone_inside['pct_change'] = (-100*(1-np.round((comp_zone_inside.eid/comp_zone_inside.week_before), 2)))#.astype(int)

sorter = ['Monday', 'Tuesday', 'Wednesday', 'Thursday', 'Friday']
comp_zone_inside.dayofweek = comp_zone_inside.dayofweek.astype("category")
comp_zone_inside.dayofweek.cat.set_categories(sorter, inplace=True)
comp_zone_inside=comp_zone_inside.sort_values('dayofweek')
comp_zone_inside.sort_values('pct_change')
comp_zone_inside.sort_values('pct_change').to_csv('data/eid_comp_zone_inside_agency.csv')
CLUSTER_TO_LINE = 'data/archive/2018-08-22/ClusterToLine.zip'

def get_cluster_to_line_df(path):
    cols = ['agency_name', 'route_id', 'route_short_name', 
            'cluster_name', 'from_date', 'to_date', 'cluster_id', 
            'route_type', 'route_type_desc', 'cluster_sub_desc', 'EXTRA']
    ctl = (pd.read_csv(path, encoding='windows-1255', 
                      skiprows=[0], header=None, names = cols)
           .drop(columns=['EXTRA']))
    return ctl

ctl = get_cluster_to_line_df(CLUSTER_TO_LINE)
ctl.route_id = ctl.route_id.astype(str)
ctl.shape
rs.groupby('date').size().head()
ctl.head()
rs[rs.date=='2018-08-22']
crs = rs.merge(ctl[['route_id', 'cluster_name', 'route_type_desc']], on='route_id', how='left')
eid_routes = crs[crs.date>'2018-08-19']
week_before_routes = crs[(crs.date<='2018-08-19') & (crs.date>'2018-08-12')]
id_num_trips_cluster = eid_routes.groupby(['dayofweek', 'cluster_name', 'route_type_desc']).num_trips.sum()
week_before_cluster = week_before_routes.groupby(['dayofweek', 'cluster_name', 'route_type_desc']).num_trips.sum()

comp_cluster = (pd.concat([week_before_cluster, id_num_trips_cluster], axis=1, keys = ['week_before', 'eid'])
             #.stack()
             .reset_index()
             .fillna(0)
             #.rename(columns={'level_2': 'week', 0: 'num_trips'})
            )
comp_cluster
comp_cluster['pct_change'] = (-100*(1-np.round((comp_cluster.eid.divide(comp_cluster.week_before, fill_value=0)), 3)))#.replace(np.inf, np.nan).dropna().astype(int)

sorter = ['Monday', 'Tuesday', 'Wednesday', 'Thursday', 'Friday', 'Saturday', 'Sunday']
comp_cluster.dayofweek = comp_cluster.dayofweek.astype("category")
comp_cluster.dayofweek.cat.set_categories(sorter, inplace=True)
comp_cluster=comp_cluster.sort_values('dayofweek')
comp_cluster
comp_cluster[comp_cluster.week_before>0].sort_values('pct_change').to_csv('data/eid_comp_cluster.csv')
alt.Chart(comp_cluster[~comp_cluster.dayofweek.isin(('Saturday', 'Sunday', 'Monday'))]).mark_point().encode(
    x = alt.X('dayofweek:N', ),
    y = 'pct_change',
    color = 'cluster_name:N',
    tooltip = 'cluster_name',
).properties(height=600, width=800)
routes_to_check = ['149','271', '300', '301', '350', '361', '369', '402',
                  '417', '433', '437', '470', '480', '500', '826',
                  '836', '845', '947']
rids_149 = ['7716', '7718']#, '7719', '7720']
rids = ['4440', '4444', '4446', '4443', '16723',
        '7393', '7395', '14180', '14181', '14183', '14184', '14185', 
        #'13690', '7396', '7398', '7400', '7402', 
        '6588', '6601', '16314', '17845', '13666', '19577', '17846',
        '4474', '4475',
        '8151', '8153', '8162', '8164', '8165',
        '17143', '6642', '16241', '16242', '16245', '6635', '16240', '16244', '6646', '6649',
        '6656', '6660', '6661', '16415',
        '6952', '6954',
        '6966', '6967', '6969',
        '7005', '7007', '7008', '7012',  '7013', '7014', '7015', '7016',
        '10958', '7020', '7022', '7023', '7024', '7026', '7027', '7028', '7030', '7033', '7034', '15337',
        '19671', '19672',
        '15123', '19731', '7072', '7083', '7081', '7079',
        '13347', '7111', '7112',
        '7159', '7161',
        '19740', '19741', '7220', '7221', '7224', '7226', '13136', '7225'
        
       ]
(week_before_routes[week_before_routes.route_short_name==routes_to_check[0]]
 .drop_duplicates(subset=['route_id', 'route_short_name', 'route_long_name'])
 #.route_id.tolist()
)
rids = rids_149
eid_rtc = eid_routes[eid_routes.route_id.isin(rids)]
wb_rtc = week_before_routes[week_before_routes.route_id.isin(rids)]

id_num_trips_zone = eid_rtc.groupby(['dayofweek', 'route_short_name']).num_trips.sum()
week_before_zone = wb_rtc.groupby(['dayofweek', 'route_short_name']).num_trips.sum()

comp_route = (pd.concat([week_before_zone, id_num_trips_zone], axis=1, keys = ['week_before', 'eid'])
             #.stack()
             .reset_index()
             #.rename(columns={'level_2': 'week', 0: 'num_trips'})
            )


comp_route['pct_change'] = (-100*(1-np.round((comp_route.eid/comp_route.week_before), 2))).astype(int)

sorter = ['Monday', 'Tuesday', 'Wednesday', 'Thursday', 'Friday', 'Saturday', 'Sunday']
comp_route.dayofweek = comp_route.dayofweek.astype("category")
comp_route.dayofweek.cat.set_categories(sorter, inplace=True)
comp_route=comp_route.sort_values('dayofweek')
comp_route[comp_route['pct_change']<0]
ts = (pd.concat([pd.read_pickle(f, compression='gzip') 
                 for f in glob.glob('data\\gtfs_stats_MOD_ffill\\2018-08-[1-2]?_trip_stats.pkl.gz')])
      .assign(date = lambda x: pd.to_datetime(x.date))
      .assign(dayofweek = lambda x: x['date'].dt.strftime('%A')))
eid_trips = ts[(ts.date>'2018-08-19') & (ts.date<'2018-08-25') & (ts.route_id.isin(rids))]
week_before_trips = ts[(ts.date<='2018-08-17') & (ts.date>'2018-08-12') & (ts.route_id.isin(rids))]
m = week_before_trips[['dayofweek', 'start_time', 'date']].merge(eid_trips[['dayofweek', 'start_time', 'date']], how='outer', on=['start_time', 'dayofweek'])
m['hour'] = m.start_time.str.split(':').apply(lambda x: int(x[0]))
m['week_before'] = m.date_x.notna()
m['eid'] = m.date_y.notna()
m.loc[(m.dayofweek=='Tuesday') & ((m.hour>=13) & (m.hour<=19))].sort_values('start_time').drop(['dayofweek', 'date_x', 'date_y', 'hour'], axis=1)

m['di'] = m.dayofweek.replace({'Monday': 2, 'Tuesday': 3, 'Wednesday': 4, 'Thursday': 5, 'Friday': 6})
m.sort_values(['di', 'start_time']).drop(['di', 'date_x', 'date_y', 'hour'], axis=1).to_csv('data/eid_149.csv', index=False)
alt.Chart(comp_route[~comp_route.dayofweek.isin(('Saturday', 'Sunday', 'Monday'))]).mark_point().encode(
    x = alt.X('dayofweek:N', ),
    y = 'pct_change',
    color = 'route_short_name:N',
    tooltip = 'route_short_name',
).properties(height=600, width=800)
id_num_trips_agency = eid_routes.groupby(['dayofweek', 'agency_name']).num_trips.sum()
week_before_agency = week_before_routes.groupby(['dayofweek', 'agency_name']).num_trips.sum()

comp_agency = (pd.concat([week_before_agency, id_num_trips_agency], axis=1, keys = ['week_before', 'eid'])
             #.stack()
             .reset_index()
             #.rename(columns={'level_2': 'week', 0: 'num_trips'})
            )


comp_agency['pct_change'] = (-100*(1-np.round((comp_agency.eid/comp_agency.week_before), 2))).astype(int)

sorter = ['Monday', 'Tuesday', 'Wednesday', 'Thursday', 'Friday', 'Saturday', 'Sunday']
comp_agency.dayofweek = comp_agency.dayofweek.astype("category")
comp_agency.dayofweek.cat.set_categories(sorter, inplace=True)
comp_agency=comp_agency.sort_values('dayofweek')
comp_agency
alt.Chart(comp_agency[~comp_agency.dayofweek.isin(('Saturday', 'Sunday', 'Monday', 'Friday'))]).mark_bar().encode(
    #color = 'dayofweek:N',
    y = 'mean(pct_change)',
    x = 'agency_name:N',
    tooltip = ['agency_name'],
).properties(height=600, width=800)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.20315674324113142, 'fraction_numerical': 0.05610251601812783, 'mean_word_length': 4.868408986703347, 'pattern_counts': {'":': 0, '<': 16, '<?xml version=': 0, '>': 17, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '2', 'dir': 'jupyter-structured-clean-dedup', 'id': '536567', 'license': 'MIT', 'n_tokens_mistral': 5438, 'n_tokens_neox': 4891, 'n_words': 693, 'path': 'openbus_19_id_al_adha.ipynb', 'repo_name': 'cjer/open-bus-explore', 'size': '12798'}