<jupyter_start><jupyter_text>Preparation of the LoS tables Saves a StepModel with shortest paths and LoS attributes for the PT network Needs aggregated PT networks<jupyter_code>input_path = '../input/'
output_path = '../output/'
model_path = '../model/'
sm = stepmodel.read_json(model_path + 'de_pt_network_agg')
ae = stepmodel.read_json(model_path + 'de_pt_access_egress')
aux = stepmodel.read_json(model_path + 'de_pt_network_ancillary')
sm.centroids = ae.centroids
sm.footpaths = ae.footpaths
sm.zone_to_transit = ae.zone_to_transit
sm.links.shape<jupyter_output><empty_output><jupyter_text>Prepare links for pathfinder<jupyter_code># Generate headways (two times the waiting time at a stop when changing between trips)
# ASSUMPTION: Take average headways for core hours between 6 a.m. and 8 p.m.
freq = aux.frequencies.loc[(aux.frequencies['hour']>=6) &
                           (aux.frequencies['hour']<=19)
                          ].groupby('stop_id').agg({'trip_id': 'sum'})
freq['trip_id'] = 14*3600 / freq['trip_id'] # 14 hours
headway_dict = freq['trip_id'].to_dict()
# ASSUMPTION: Maximum waiting time is 20 minutes, minumum is 5 minutes
sm.links['headway'] = sm.links['a'].map(headway_dict)
sm.links['headway'] = sm.links['headway'].replace(
    {np.nan: 15*60*2}).clip(lower=5*60*2, upper=20*60*2)
# ASSUMPTION: Air trips require security checks and so on...
sm.links.loc[sm.links['route_type']=='air', 'headway'] = 45*60*2
sm.links['headway'] = sm.links['headway'].astype(int)
sm.links['headway'].hist(bins=100, figsize=(4,4), log=True)
# Reset access/egress time to zero
# Because it should not be a mode choice decision factor.
# LoS attribute accessibility will handle this.
#sm.zone_to_transit['time'] = 0
# Generate length for PT links as the crow flies
if 'length' not in sm.links.columns:
    tqdm.pandas()
    sm.links['length'] = sm.links['geometry'].progress_apply(
        lambda l: int(geodesic(l.coords[0], l.coords[-1]).m))
sm.links.sample()
# Load OD pairs from volumes if available
od_set = None
try:
    v = stepmodel.read_json(model_path + 'de_volumes')
    od_set = set([tuple(l) for l in v.volumes[['origin', 'destination']].values.tolist()])
    print('Generated OD set with {} pairs'.format(len(od_set)))
except:
    print('The volumes file is not (yet) available')<jupyter_output>Generated OD set with 160801 pairs<jupyter_text>Apply PT pathfinderIt generates the LoS table with shortest paths for each origin destination pair<jupyter_code>'''This cell would run all possible mode combinations
sm.step_pt_pathfinder(
    broken_routes=False,
    broken_modes=True, # Allow different mode combinations
    route_column='route_id',
    mode_column='route_type',
    boarding_time=0, # Assumption
    alighting_time=0,
    speedup=True,
    walk_on_road=False,
    keep_pathfinder=False, # Saves it as attribute for multiple iterations
    force=False, # If indices are inconsistent
    path_analysis=True, # Create path, time and length columns
    drop_duplicates=True, # Considering duplicate paths
    cutoff=np.inf, # distance limit between points for Dijkstra algorithm
    od_set=od_set) # Exclude OD pairs that are not used'''
# We don't want the algorithm to search paths for all mode combinations.
# Thus, we have to extract the pathfinder and search them manually.
ppf = PublicPathFinder(sm)
# Now, build custom mode combinations
# People just don't go by coach and then switch to rail or airplane within Germany
ppf.build_mode_combinations(mode_column='route_type')
ppf.mode_combinations = [x for x in ppf.mode_combinations
                         if not ('coach' in x and
                         ('rail_long' in x or
                          'rail_short' in x or
                          'air' in x))]
ppf.mode_combinations
# Run the pathfinder for all mode combinations
# ASSUMPTION: Include a penalty for people to choose less transfers
transfer_penalty = 0#20*60
ppf.find_broken_mode_paths(workers=3, # CPU cores
                           cutoff=np.inf,
                           od_set=od_set,
                           mode_column='route_type',
                           boarding_time=transfer_penalty)
ppf.broken_mode_paths.sample()
# Run the path analysis and assign paths as LoS table
sm.pt_los = analysis.path_analysis_od_matrix(
    od_matrix=ppf.broken_mode_paths,
    links=sm.links,
    nodes=sm.nodes,
    centroids=sm.centroids)
print(len(sm.pt_los.index))
sm.pt_los.loc[sm.pt_los['origin']!=sm.pt_los['destination']].sample(3)<jupyter_output>2744932<jupyter_text>Add attributes to LoS table<jupyter_code># Disaggregate time into in-vehicle and access/egress
sm.analysis_pt_time(boarding_time=0)
print(sm.pt_los.columns)
# Add price, parametrisation comes later
sm.pt_los['price'] = 0
# Add distances
len_dict = sm.links['length'].to_dict()
sm.pt_los['length'] = sm.pt_los['link_path'].apply(lambda p: sum([len_dict[l] for l in p]))
sm.pt_los[['ntransfers', 'length']+[c for c in sm.pt_los.columns if c[-4:]=='time']].sample(2)
# Add route_types
type_dict = sm.links['route_type'].to_dict()
sm.pt_los['route_types'] = [set([type_dict[link] for link in l])
                            for l in list(sm.pt_los['link_path'])]
# Add main route_type based on a hierarchy
hierarchy = ['air', 'coach', 'rail_long', 'rail_short', 'bus', 'walk']
def main_type(types):
    t = hierarchy[-1]
    try:
        t = hierarchy[min([hierarchy.index(t) for t in types])]
    except ValueError: # empty types set
        pass
    return t
sm.pt_los['route_type'] = sm.pt_los['route_types'].apply(main_type)
# Add number of transfers
#sm.pt_los['ntransfers'] = sm.pt_los['boardings'].apply(lambda l: max(0, len(l) - 1))
sm.pt_los.sample(2)
# Make a copy before restricting results
los = sm.pt_los.copy()
#sm.pt_los = los.copy()<jupyter_output><empty_output><jupyter_text>Clean PT LOS<jupyter_code># Berlin - Hamburg
all_pt_paths(sm, 'DE300', 'DE600', verbose=False, group_name='route_type')
# Drop trips with coach-rail or coach-air combinations
sm.pt_los = sm.pt_los.loc[~sm.pt_los['route_types'].apply(
    lambda x: 'coach' in x and ('rail_long' in x or 'rail_short' in x or 'air' in x))]
sm.pt_los.shape
# Drop unrealistically long trips
sm.pt_los = sm.pt_los.loc[sm.pt_los['in_vehicle_time']<10*3600]
sm.pt_los.shape
# Drop duplicates
sm.pt_los['path'] = sm.pt_los['path'].apply(tuple)
sm.pt_los.drop_duplicates('path', inplace=True)
sm.pt_los.reset_index(inplace=True)
sm.pt_los.shape
# Time distribution
sm.pt_los.hist(column='time', bins=100, figsize=(4,4), log=True)
sm.pt_los.hist(column='ntransfers', figsize=(4,4))
# Main mode distribution by transfers
t = sm.pt_los.copy()
t['bins'] = pd.cut(t['ntransfers'], bins=[0,1,3,5,10,20,100],
                   labels=["0-1", "2-3","4-5","6-10","11-20","21+"])
axs = t.groupby(['route_type','bins']).size().unstack(fill_value=0).plot.pie(
    subplots=True, figsize=(16, 3),
    startangle=0, counterclock=False, title='Main mode by number of transfers',
    legend=False, labeldistance=None)
axs[0].legend(loc='lower left')
# Distribution of mode combinations
t['route_types'] = t['route_types'].apply(tuple)
t.groupby('route_types').count()['route_type'].plot(kind='bar')
t['time'] = t['time'] / 60
t['bins'] = pd.cut(t['time'], bins=[0,15,30,60,120,360,500],
                   labels=["0-15", "16-30","31-60","61-120","120-360","361+"])
axs = t.groupby(['route_type','bins']).size().unstack(fill_value=0).plot.pie(
    subplots=True, figsize=(16, 3),
    startangle=0, counterclock=False, title='Main mode by trip duration',
    legend=False, labeldistance=None)
axs[3].legend()
t = None
# Drop unnecessary long connections if there is a waiting time applied
if sm.links['headway'].mean() > 5*60:
    sm.pt_los = sm.pt_los.loc[sm.pt_los['ntransfers']<=10]
    print(sm.pt_los.shape)
    # Drop coach connections with more than one transfer
    sm.pt_los = sm.pt_los.loc[~((sm.pt_los['route_type']=='coach') &
                                (sm.pt_los['boardings'].apply(
                                    lambda l: len([n for n in l if n.startswith('coach')])>2)
                                ))]
    print(sm.pt_los.shape)
# Keep only the fastest trip(s)
sm.pt_los['route_types'] = sm.pt_los['route_types'].apply(tuple)
sm.pt_los = sm.pt_los.sort_values('time').groupby(
    ['origin', 'destination', 'route_types']).nth([0, 1]).reset_index()
sm.pt_los.shape
sm.pt_los.groupby('route_types').count()['route_type'].plot(kind='bar')
# Berlin - Hamburg
all_pt_paths(sm, 'DE300', 'DE600', verbose=False, group_name='route_type')<jupyter_output><empty_output><jupyter_text>Save model<jupyter_code># Ditch unneccessary columns
to_drop = ['gtime', 'path', 'reversed', 'pathfinder_session',
           'broken_modes', 'boardings', 'alightings', 'node_path',
           'footpaths', 'ntlegs', 'transfers', 'boarding_links',
           'alighting_links', 'all_walk', 'time_link_path',
           'length_link_path', 'boarding_time', 'index']
sm.pt_los = sm.pt_los.drop(to_drop, axis=1, errors='ignore')
# Clean links table
to_drop = ['duration', 'cost', 'origin', 'destination']
sm.links = sm.links.drop(to_drop, axis=1, errors='ignore')
# Make table smaller
cols = ['time', 'access_time', 'in_vehicle_time', 'footpath_time',
        'waiting_time', 'length']
sm.pt_los[cols] = sm.pt_los[cols].astype(int)
sm.pt_los.sample()
# Saving LoS...
sm.to_json(model_path + 'de_pt_los', only_attributes=['pt_los'],
           encoding='utf-8')
# Split links in graph and auxiliary information
# for file sizes being compatible with github's size limit
cols = ['link_sequence', 'route_id', 'time', 'trip_id', 'length', 'headway']
auxiliary = sm.links[cols]
sm.links.drop(cols, axis=1, inplace=True)
sm.to_json(model_path + 'de_pt_network_agg', only_attributes=[
    'zones', 'links', 'nodes', 'pt_route_types'],
           encoding='utf-8')
# Save auxiliary information seperately
auxiliary['index'] = auxiliary.index
auxiliary.reset_index(drop=True, inplace=True)
auxiliary.to_json(model_path + 'de_pt_network_agg/links_quetzaldata.json')<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1471698113207547, 'fraction_numerical': 0.01906653426017875, 'mean_word_length': 4.4, 'pattern_counts': {'":': 0, '<': 26, '<?xml version=': 0, '>': 26, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 1, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '6', 'dir': 'jupyter-structured-clean-dedup', 'id': '461000', 'license': 'Apache-2.0', 'n_tokens_mistral': 3634, 'n_tokens_neox': 3445, 'n_words': 786, 'path': 'quetzal_germany-master/notebooks/prep30_los_pt.ipynb', 'repo_name': 'avi-shake/Flensburg_Modeling', 'size': '10070'}