<jupyter_start><jupyter_text>Load shapes<jupyter_code>gdf = gpd.read_file('input_data/tl_2016_us_uac10').sort_values(by='ALAND10', ascending=True)
len(gdf)
# remove the tiny "urban clusters" to retain only "urban areas"
urban_areas = gdf[~gdf['NAMELSAD10'].str.contains('Urban Cluster')]
len(urban_areas)
# tigerline data is epsg:4269, but osm uses epsg:4326, so project it
print(urban_areas.crs)
urban_areas = urban_areas.to_crs({'init':'epsg:4326'})
urban_areas.crs
urban_areas[['ALAND10', 'NAMELSAD10', 'GEOID10']].tail()
# optionally buffer so streets that form the boundary don't get ignored
# but, buffering will mess up using ALAND for area as polygon area will capture nodes outside ALAND!
if buffer:
    buffer_dist = 50
    urban_areas.name = 'urban_areas'
    urban_areas_utm = ox.project_gdf(urban_areas)
    urban_areas_utm['geometry'] = urban_areas_utm['geometry'].buffer(buffer_dist)
    urban_areas_buffered = ox.project_gdf(urban_areas_utm, to_latlong=True)
    print('Buffered geometry.')
urban_areas['geometry'].iloc[0]<jupyter_output><empty_output><jupyter_text>Begin process<jupyter_code>queries = urban_areas.apply(lambda row: {'name':row['NAMELSAD10'],
                                         'geoid':row['GEOID10'],
                                         'name_geoid':'{}_{}'.format(row['NAMELSAD10'], row['GEOID10']).replace(',', '_').replace(' ', '_'),
                                         'polygon':row['geometry']}, axis=1).tolist()
queries[0]
start_time = time.time()

for query in queries:
    try:
        if not os.path.exists('G:/Geoff/osmnx/urbanized-areas-usa/{}.graphml'.format(query['name_geoid'])):
            G = ox.graph_from_polygon(polygon=query['polygon'], network_type=network_type, 
                                      name=query['name_geoid'], retain_all=retain_all)
            ox.save_graph_shapefile(G, filename=query['name_geoid'])
            ox.save_graphml(G, filename='{}.graphml'.format(query['name_geoid']))
    except Exception as e:
        print('{} failed: {}'.format(query['name_geoid'], e))
        
print('Finished making graphs in {:,.2f} seconds'.format(time.time() - start_time))
#41,000 seconds last time<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.14654002713704206, 'fraction_numerical': 0.02125734961555857, 'mean_word_length': 3.9707865168539325, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 7, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '2', 'dir': 'jupyter-structured-clean-dedup', 'id': '22839', 'license': 'MIT', 'n_tokens_mistral': 800, 'n_tokens_neox': 746, 'n_words': 130, 'path': 'urbanized-areas-usa/get_every_us_urban_area_graph.ipynb', 'repo_name': 'gboeing/street-network-analysis', 'size': '2211'}