<jupyter_start><jupyter_text>Preparation of the transport network. Saves access and egress links for each zone. Needs all networks.<jupyter_code>input_path = '../input_static/'
output_path = '../output/'
model_path = '../model/'
sm = stepmodel.read_json(input_path + 'de_pt_network')
bus = stepmodel.read_json(input_path + 'de_pt_network_bus')
road = stepmodel.read_json(input_path + 'de_road_network')<jupyter_output><empty_output><jupyter_text>Clean zones<jupyter_code># Drop unused zones for later steps to work 
print(len(sm.zones.index))
sm.zones = sm.zones.loc[sm.zones['NUTS_ID'].str.startswith('DE')]
sm.zones = sm.zones[sm.zones["LEVL_CODE"]==3]
print(len(sm.zones.index))
# Compute controids
sm.preparation_ntlegs(
    zone_to_transit=False,
    zone_to_road=False)<jupyter_output><empty_output><jupyter_text>Assumptions<jupyter_code># Walking vs. driving to the train station
threshold = 500 # in m
speed_non_motorised_modes = 17 # in km/h
speed_car = 50 # in km/h
speed_footpaths = 5 # in km/h, assumed
# Max. distances for accessing the transport nodes
# in m
threshold_dict = {'road': 10000,
                  'road_pt': 500,
                  'pt': 12000,
                  'footpaths': 500,
                  'footpaths_air': 2200,
                  'cycling_zones': 20000}
# When having centroids computed as geometric centers of
# political zones, it can happen that centroids come very
# close to each other. This requires special filtering
# of connectors for not making routes unrealistically short
def filter_connectors(df, keep=2, length_weight=None, dist_centr_weight=None,
                      conn_weight=None, conn_dict=None):
    # Each link exists as both access and egress
    ac = df.loc[df['direction']=='access']
    eg = df.loc[df['direction']=='eggress']
    analysis_cols = []
    if length_weight:
        # Link length (in km) weighted
        ac['length'] = - np.power(ac['distance'] / 1e3, length_weight)
        eg['length'] = - np.power(eg['distance'] / 1e3, length_weight)
        analysis_cols.append('length')
    if dist_centr_weight:
        # Calculate distance to the nearest centroid (except own one)
        ac['centr_dist'] = np.power([min([p.distance(c) for c in list(
            sm.centroids.loc[sm.centroids['FID']!=own_c, 'geometry'])]
                                        ) / 1.4e-2 for own_c, p in zip(
            list(ac['a']), [geometry.Point(l.coords[-1]) for l in ac['geometry']])],
                                    dist_centr_weight)
        eg['centr_dist'] = np.power([min([p.distance(c) for c in list(
            sm.centroids.loc[sm.centroids['FID']!=own_c, 'geometry'])]
                                        ) / 1.4e-2 for own_c, p in zip(
            list(eg['b']), [geometry.Point(l.coords[0]) for l in eg['geometry']])],
                                    dist_centr_weight)
        analysis_cols.append('centr_dist')
    if conn_weight:
        # Calculate connectivity
        ac['connectivity'] = np.power(ac['b'].map(conn_dict), conn_weight)
        eg['connectivity'] = np.power(eg['a'].map(conn_dict), conn_weight)
        analysis_cols.append('connectivity')
    # Calculate link performance with given attributes
    ac['sum'] = ac[analysis_cols].sum(axis=1)
    eg['sum'] = eg[analysis_cols].sum(axis=1)
    # Get the n most wanted links
    ac = ac.sort_values('sum', ascending=False).groupby('a').head(keep)
    eg = eg.sort_values('sum', ascending=False).groupby('b').head(keep)
    return ac.append(eg).reset_index()[df.columns]<jupyter_output><empty_output><jupyter_text>Generate footpaths table<jupyter_code>sm.nodes.shape
# The Voronoi algorithm is too slow for this amount of nodes
#sm.footpaths = connectivity.build_footpaths(
#    sm.nodes,
#    speed=speed_footpaths,
#    max_length=threshold_dict['footpaths'],
#    n_clusters=1000,
#    coordinates_unit=sm.coordinates_unit)
# Generate footpaths between modes
sm.footpaths = sm.footpaths.iloc[0:0]
for o in ['rail_short_distance']:
    for d in [t for t in sm.pt_route_types if t!='bus']:
        if o != d:
            ntlegs = engine.ntlegs_from_centroids_and_nodes(
                sm.nodes.loc[sm.nodes['route_type']==o],
                sm.nodes.loc[sm.nodes['route_type']==d],
                short_leg_speed=speed_footpaths,
                long_leg_speed=speed_non_motorised_modes,
                threshold=threshold,
                n_neighbors=1,
                coordinates_unit=sm.coordinates_unit)
            if d == 'air':
                ntlegs = ntlegs.loc[ntlegs['distance']<=
                                    threshold_dict['footpaths_air']]
            else:
                ntlegs = ntlegs.loc[ntlegs['distance']<=
                                    threshold_dict['footpaths']]
            sm.footpaths = sm.footpaths.append(ntlegs)
len(sm.footpaths.index)
# Number of nodes that overlay each other
sm.footpaths.loc[sm.footpaths['distance']==0].shape
# Add bus connections
bus_foot = sm.footpaths.iloc[0:0]
for d in [t for t in sm.pt_route_types if t!='bus']:
    ntlegs = engine.ntlegs_from_centroids_and_nodes(
        bus.nodes,
        sm.nodes.loc[sm.nodes['route_type']==d],
        short_leg_speed=speed_footpaths,
        long_leg_speed=speed_non_motorised_modes,
        threshold=threshold,
        n_neighbors=1,
        coordinates_unit=sm.coordinates_unit)
    if d == 'air':
        ntlegs = ntlegs.loc[ntlegs['distance']<=
                            threshold_dict['footpaths_air']]
    else:
        ntlegs = ntlegs.loc[ntlegs['distance']<=
                            threshold_dict['footpaths']]
    bus_foot = bus_foot.append(ntlegs)
len(bus_foot.index)
# Keep only bus footpaths to the most connected stop(s) in each zone
n_links_dict = sm.links[['a', 'b']].stack().value_counts().to_dict()
sm.nodes['n_links'] = [n_links_dict[i] if i in n_links_dict.keys() else 0
                       for i in list(sm.nodes.index)]
sm.nodes['index'] = sm.nodes.index
node_list = list(sm.nodes.sort_values('n_links', ascending=False).groupby(
    ['route_type', 'FID']).nth([0,1,2])['index'])
bus_foot = bus_foot.loc[((bus_foot['direction']=='access')&(bus_foot['b'].isin(node_list))) |
                        ((bus_foot['direction']=='eggress')&(bus_foot['a'].isin(node_list)))]
len(bus_foot)
sm.footpaths = sm.footpaths.append(bus_foot)
# Generate footpaths between centroids
ntlegs = engine.ntlegs_from_centroids_and_nodes(
    sm.centroids,
    sm.centroids,
    short_leg_speed=speed_non_motorised_modes,
    long_leg_speed=speed_non_motorised_modes,
    threshold=threshold,
    n_neighbors=2,
    coordinates_unit=sm.coordinates_unit)
ntlegs = ntlegs.loc[ntlegs['distance']<=
                    threshold_dict['cycling_zones']]
ntlegs = ntlegs.loc[ntlegs['distance']!=0]
ntlegs.drop_duplicates(['direction', 'distance', 'time'], inplace=True)
sm.footpaths = sm.footpaths.append(ntlegs)
ntlegs.shape
# Reindex
sm.footpaths.reset_index(drop=True, inplace=True)
sm.footpaths.index = 'foot_' + pd.Series(sm.footpaths.index).astype(str)
sm.footpaths.sample(n=3)<jupyter_output><empty_output><jupyter_text>Add access and egress links from zone centroids<jupyter_code># How many links does every node have?
sm.links[['a', 'b']].stack().value_counts().hist(bins=100, log=True)
# Compute road access and egress links
sm.zone_to_road = engine.ntlegs_from_centroids_and_nodes(
    sm.centroids,
    road.road_nodes,
    short_leg_speed=speed_footpaths,
    long_leg_speed=speed_non_motorised_modes,
    threshold=threshold,
    n_neighbors=20,
    coordinates_unit=sm.coordinates_unit)
sm.zone_to_road = sm.zone_to_road.loc[
    sm.zone_to_road['distance']<=threshold_dict['road']]
len(sm.zone_to_road.index)
sm.zone_to_road = filter_connectors(sm.zone_to_road, keep=1,
                                    length_weight=None,
                                    dist_centr_weight=1)
len(sm.zone_to_road.index)
sm.zones.loc[~sm.zones['FID'].isin(list(sm.zone_to_road['a']))]
# Compute road - PT links
sm.road_to_transit = engine.ntlegs_from_centroids_and_nodes(
    sm.nodes,
    road.road_nodes,
    short_leg_speed=speed_footpaths,
    long_leg_speed=speed_non_motorised_modes,
    threshold=threshold,
    n_neighbors=1,
    coordinates_unit=sm.coordinates_unit)
sm.road_to_transit = sm.road_to_transit.loc[
    sm.road_to_transit['distance']<=threshold_dict['road_pt']]
len(sm.road_to_transit.index)
# Every route type is saved once in this attribute
assert len(sm.pt_route_types) == len(sm.links['route_type'].unique())
sm.pt_route_types
# How many links does every node have?
n_links_dict = sm.links[['a', 'b']].append(bus.links[['a', 'b']]).append(
    sm.footpaths[['a', 'b']]).stack().value_counts().to_dict()
# Compute PT access and egress links by route type
sm.zone_to_transit = sm.zone_to_transit.iloc[0:0]
for t in sm.pt_route_types:
    ntlegs = engine.ntlegs_from_centroids_and_nodes(
        sm.centroids,
        sm.nodes.loc[sm.nodes['route_type']==t],
        short_leg_speed=speed_footpaths,
        long_leg_speed=speed_car, # Take the car for longer trips
        threshold=threshold,
        n_neighbors=5,
        coordinates_unit=sm.coordinates_unit)
    # Cut off long links
    ntlegs = ntlegs.loc[ntlegs['distance']<=
                        threshold_dict['pt']]
    ntlegs['route_type'] = t
    sm.zone_to_transit = sm.zone_to_transit.append(filter_connectors(
        ntlegs, keep=2, length_weight=1, dist_centr_weight=2,
        conn_weight=.4, conn_dict=n_links_dict)).append(filter_connectors(
            ntlegs, keep=2, length_weight=1, conn_weight=1, conn_dict=n_links_dict))
sm.zone_to_transit.reset_index(drop=True, inplace=True)
len(sm.zone_to_transit.index)
# Add bus connections
ntlegs = engine.ntlegs_from_centroids_and_nodes(
    sm.centroids,
    bus.nodes,
    short_leg_speed=speed_footpaths,
    long_leg_speed=speed_car, # Take the car for longer trips
    threshold=threshold,
    n_neighbors=5, # Generate a bunch and clean later
    coordinates_unit=sm.coordinates_unit)
# Cut off long links
ntlegs = ntlegs.loc[ntlegs['distance']<=
                    threshold_dict['pt']]
ntlegs['route_type'] = 'bus'
len(ntlegs.index)
sm.zone_to_transit = sm.zone_to_transit.append(filter_connectors(
    ntlegs, keep=1, length_weight=1, dist_centr_weight=2,
    conn_weight=.4, conn_dict=n_links_dict)).append(filter_connectors(
        ntlegs, keep=1, length_weight=1, conn_weight=1, conn_dict=n_links_dict))
sm.zone_to_transit.reset_index(drop=True, inplace=True)
len(sm.zone_to_transit.index)
sm.zone_to_transit.drop_duplicates(['a', 'b', 'direction'], inplace=True)
sm.zone_to_transit.reset_index(drop=True, inplace=True)
len(sm.zone_to_transit.index)
# Number of legs by route type
sm.zone_to_transit.groupby(['route_type', 'direction'])['a'].count()
# Every zone must have an access and an egress link to PT
#sm.zones.loc[sm.zones['FID'].isin(list(sm.zone_to_transit['a']))==False]
assert sm.zones['FID'].isin(list(sm.zone_to_transit['a'])).all()
assert sm.zones['FID'].isin(list(sm.zone_to_transit['b'])).all()<jupyter_output><empty_output><jupyter_text>Parametrise access and egress links<jupyter_code># Road - PT connectors
sm.road_to_transit['distance'] = 0
sm.road_to_transit['time'] = 5*60 # in seconds
sm.road_to_transit.sample(n=2)
# Road - centroid connectors
sm.zone_to_road['distance'] = 0
sm.zone_to_road['time'] = 0
sm.zone_to_road.sample(n=2)
# Load distances to PT stops from census data
clusters = pd.read_csv('../input/spatial_census_refined_FID.csv')
clusters.set_index('index', inplace=True)
dist_col = 'weighted_dist'
mean_dist_col = 'mean_weighted_dist'
cent_col = 'cluster_center'
#pop_col = 'population'
node_col = 'node_id'
clusters['NUTS_ID'] = clusters['FID'].apply(lambda s: str(s).split('_')[0])
# Create a zone - distance - type dict
# Take the mean weighted distance, assume population shares are about equal
zone_type_dist = clusters.groupby(['NUTS_ID', 'route_type']).agg(
    {mean_dist_col: 'mean'})#.to_dict()[mean_dist_col]
zone_type_dist
# Split legs into access and egress
access = sm.zone_to_transit.loc[sm.zone_to_transit['direction']=='access'].copy()
egress = sm.zone_to_transit.loc[sm.zone_to_transit['direction']=='eggress'].copy()# name in quetzal
# Alter the distance
route_types = clusters['route_type'].unique()
for t in route_types:
    for zone in list(sm.zones['NUTS_ID']):
        try:
            access.loc[(access['route_type']==t) & (access['a']==zone),
                       'distance'] = zone_type_dist.loc[(zone, t), mean_dist_col]
        except KeyError:
            # There is no such station in this zone
            access.loc[(access['route_type']==t) & (access['a']==zone),
                       'distance'] = np.nan
for t in route_types:
    for zone in list(sm.zones['NUTS_ID']):
        try:
            egress.loc[(egress['route_type']==t) & (egress['b']==zone),
                       'distance'] = zone_type_dist.loc[(zone, t), mean_dist_col]
        except KeyError:
            # There is no such station in this zone
            egress.loc[(egress['route_type']==t) & (egress['b']==zone),
                       'distance'] = np.nan
sm.zone_to_transit = access.append(egress)
len(sm.zone_to_transit.loc[sm.zone_to_transit['distance'].isna()])
sm.zone_to_transit['speed'].hist(bins=20)
# Correct the speed by linear interpolation
sm.zone_to_transit['speed'] = sm.zone_to_transit['distance'].apply(
    lambda d: (d-threshold)/1000 * \
        ((speed_car-speed_footpaths) / ((threshold_dict['pt']-threshold)/1000))
).clip(lower=speed_footpaths, upper=speed_car)
# Alter time of PT legs
sm.zone_to_transit['time'] = sm.zone_to_transit['distance'] / \
    sm.zone_to_transit['speed'] * 3.6 # in s
sm.zone_to_transit['time'].replace({np.nan: threshold_dict['pt']/speed_car}, inplace=True)
sm.zone_to_transit.loc[sm.zone_to_transit['distance'].isna(), 'distance'] = 0
sm.zone_to_transit.loc[sm.zone_to_transit['speed'].isna(), 'speed'] = 0
print(len(sm.zone_to_transit))
sm.zone_to_transit.sample()
sm.zone_to_transit['time'].hist(bins=100)
# Ditch all legs which take too long
sm.zone_to_transit = sm.zone_to_transit.loc[sm.zone_to_transit['time']<3600]
len(sm.zone_to_transit)
# Every zone must have an access and an egress link to PT
assert sm.zones['FID'].isin(list(sm.zone_to_transit['a'])).all()
assert sm.zones['FID'].isin(list(sm.zone_to_transit['b'])).all()
# Correct link IDs in zone_to_transit for rail modes
type_dict = {'rail_short_distance': 'rail_short', 'rail_long_distance': 'rail_long'}
sm.zone_to_transit['route_type'].replace(type_dict, inplace=True)
sm.zone_to_transit['a'] = sm.zone_to_transit['a'].apply(lambda n: n.replace('rail_long_node', 'r_l_n'))
sm.zone_to_transit['a'] = sm.zone_to_transit['a'].apply(lambda n: n.replace('rail_short_node', 'r_s_n'))
sm.zone_to_transit['b'] = sm.zone_to_transit['b'].apply(lambda n: n.replace('rail_long_node', 'r_l_n'))
sm.zone_to_transit['b'] = sm.zone_to_transit['b'].apply(lambda n: n.replace('rail_short_node', 'r_s_n'))<jupyter_output><empty_output><jupyter_text>Save model<jupyter_code># Drop unneccessary columns
cols = ['speed_factor', 'short_leg_speed', 'long_leg_speed', 'rank']
sm.footpaths.drop(cols, axis=1, inplace=True, errors='ignore')
sm.zone_to_transit.drop(cols, axis=1, inplace=True, errors='ignore')
sm.zone_to_road.drop(cols, axis=1, inplace=True, errors='ignore')
sm.road_to_transit.drop(cols, axis=1, inplace=True, errors='ignore')
sm.zone_to_transit.drop(['n_links'], axis=1, inplace=True, errors='ignore')
# Make tables lighter
cols = ['distance', 'speed', 'time']
sm.footpaths[cols] = sm.footpaths[cols].astype(int)
sm.zone_to_transit[cols] = sm.zone_to_transit[cols].astype(int)
sm.zone_to_road[cols] = sm.zone_to_road[cols].astype(int)
sm.road_to_transit[cols] = sm.road_to_transit[cols].astype(int)
sm.footpaths.sample()
# Saving model...
tables = ['centroids', 'footpaths', 'zone_to_transit']
sm.to_json(model_path + 'de_pt_access_egress',
           only_attributes=tables, encoding='utf-8')
sm.to_json(model_path + 'de_road_access_egress',
           only_attributes=['centroids', 'zone_to_road', 'road_to_transit'],
           encoding='utf-8')<jupyter_output>to_hdf(overwriting): 100%|█████████████████████████████████████████████████████████████| 37/37 [00:03<00:00, 11.15it/s]
{'fraction_non_alphanumeric': 0.14597150577253745, 'fraction_numerical': 0.009150085973962172, 'mean_word_length': 4.108218318695107, 'pattern_counts': {'":': 0, '<': 39, '<?xml version=': 0, '>': 28, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 10, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '7', 'dir': 'jupyter-structured-clean-dedup', 'id': '642121', 'license': 'Apache-2.0', 'n_tokens_mistral': 5987, 'n_tokens_neox': 5759, 'n_words': 974, 'path': 'quetzal_germany-master/notebooks/prep20_access_egress.ipynb', 'repo_name': 'avi-shake/Flensburg_Modeling', 'size': '16284'}