<jupyter_start><jupyter_text>Download and process the Bay Area's walkable network<jupyter_code>import time
import os, zipfile, requests, pandas as pd, geopandas as gpd, osmnx as ox, networkx as nx

ox.config(use_cache=True, log_console=True)
print('ox {}\nnx {}'.format(ox.__version__, nx.__version__))
start_time = time.time()
# point to the shapefile for counties
counties_shapefile_url = 'http://www2.census.gov/geo/tiger/GENZ2016/shp/cb_2016_us_county_500k.zip'

# identify bay area counties by fips code
bayarea = {'Alameda':'001',
           'Contra Costa':'013',
           'Marin':'041',
           'Napa':'055',
           'San Francisco':'075',
           'San Mateo':'081',
           'Santa Clara':'085',
           'Solano':'095',
           'Sonoma':'097'}<jupyter_output><empty_output><jupyter_text>Download and extract the counties shapefile if it doesn't already exist, then load itTo use OSMnx, we need a polygon of the Bay Area's nine counties. So, we'll download a shapefile from the census, extract our counties, and take the union to form a polygon. Also, project the polygon so we can calculate its area for density stats.<jupyter_code>counties_shapefile_zip = counties_shapefile_url[counties_shapefile_url.rfind('/') + 1 :]
counties_shapefile_dir = counties_shapefile_zip[: counties_shapefile_zip.rfind('.zip')]
if not os.path.exists(counties_shapefile_dir):
    response = requests.get(counties_shapefile_url)
    with open(counties_shapefile_zip, 'wb') as f:
        f.write(response.content)
        with zipfile.ZipFile(counties_shapefile_zip, 'r') as zip_file:
            zip_file.extractall(counties_shapefile_dir)
    os.remove(counties_shapefile_zip)
counties = gpd.read_file(counties_shapefile_dir)
len(counties)
# retain only those tracts that are in the bay area counties
mask = (counties['STATEFP'] == '06') & (counties['COUNTYFP'].isin(bayarea.values()))
gdf_bay = counties[mask]
len(gdf_bay)
bayarea_polygon = gdf_bay.unary_union
bayarea_polygon
# get the convex hull, otherwise we'll cut out bridges over the bay
bayarea_polygon_hull = bayarea_polygon.convex_hull
bayarea_polygon_hull_proj, crs = ox.project_geometry(bayarea_polygon_hull)
# project by a mile to get connectivities surrounding our O-Ds
bayarea_polygon_hull_proj_buff = bayarea_polygon_hull_proj.buffer(1600) #1 mile in meters
bayarea_polygon_hull_buff, crs = ox.project_geometry(bayarea_polygon_hull_proj_buff, crs=crs, to_latlong=True)<jupyter_output><empty_output><jupyter_text>Download the street networkNow we've got our polygon of the buffered convex hull around the nine county bay area. Use OSMnx to download the street network (walkable paths) and simplify its topology. Setting `retain_all=False` means we only keep the largest (weakly) connected component. In the bay area, this means we keep only the southern 3/4 of the network. The northern part is not connected.<jupyter_code># download full walkable network and simplify its topology
G = ox.graph_from_polygon(bayarea_polygon_hull_buff, network_type='walk',
                          simplify=True, retain_all=False)
# verify that our graph is strongly connected, should be if it's a walkable network and we
# already retained only the largest weakly connected component
assert nx.is_strongly_connected(G)
# create a unique ID for each edge because osmid can hold multiple values due to topology simplification
i = 0
for u, v, k, d in G.edges(data=True, keys=True):
    d['uniqueid'] = i
    i += 1<jupyter_output><empty_output><jupyter_text>See some descriptive stats then save to disk<jupyter_code>print(len(G.nodes()))
print(len(G.edges()))
# see some basic network stats
# note, areas/densities include water
pd.Series(ox.basic_stats(G, area=bayarea_polygon_hull_proj_buff.area))
# save nodes and edges list as csv
nodes, edges = ox.graph_to_gdfs(G, node_geometry=False, fill_edge_geometry=False)

ecols = ['uniqueid', 'u', 'v', 'key', 'oneway', 'highway', 'name', 'length',
         'lanes', 'width', 'est_width', 'maxspeed', 'access', 'service',
         'bridge', 'tunnel', 'area', 'junction', 'osmid', 'ref']

edges = edges.drop(columns=['geometry']).reindex(columns=ecols)
nodes = nodes.reindex(columns=['osmid', 'x', 'y', 'ref', 'highway'])

nodes.to_csv('data/walk_network_connected/bay_area_walk_nodes.csv', index=False, encoding='utf-8')
edges.to_csv('data/walk_network_connected/bay_area_walk_edges.csv', index=False, encoding='utf-8')
# save as graphml for re-using later
ox.save_graphml(G, filename='bayarea_walk_simplified.graphml', folder='data/walk_network_connected')
# save as shapefile for GIS
ox.save_graph_shapefile(G, filename='bayarea_walk_simplified', folder='data/walk_network_connected')
# visualize the network
fig, ax = ox.plot_graph(G, node_size=0, edge_linewidth=0.2)
time.time() - start_time<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.10920016576875259, 'fraction_numerical': 0.011603812681309573, 'mean_word_length': 5.5584239130434785, 'pattern_counts': {'":': 0, '<': 17, '<?xml version=': 0, '>': 17, 'https://': 0, 'lorem ipsum': 0, 'www.': 1, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 6, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '4', 'dir': 'jupyter-structured-clean-dedup', 'id': '22612', 'license': 'BSD-3-Clause', 'n_tokens_mistral': 1715, 'n_tokens_neox': 1554, 'n_words': 430, 'path': 'network/download-bay-area-network-for-bpr_walk_connected.ipynb', 'repo_name': 'ual/DOE-repo-deliverable', 'size': '4826'}