<jupyter_start><jupyter_text>Ranch Demo NotebookThis notebook has examples of how to make Ranch calls to run through the complete network building pipeline to develop standard roadway and transit network from scratch<jupyter_code>import os
import pickle
import pandas as pd
import geopandas as gpd
from pyproj import CRS
import json
import sys

import ranch
from ranch import sharedstreets
from ranch import Roadway
from ranch import Transit
from ranch import Parameters
from ranch.utils import link_df_to_geojson, point_df_to_geojson
from ranch.logger import RanchLogger
%load_ext autoreload
%autoreload 2<jupyter_output><empty_output><jupyter_text>Remote I/OUser to update, project directory is not necessarily the ranch directory<jupyter_code># project directory
root_dir = os.path.join("D:/merced")

external_dir = os.path.join(root_dir, "data", "external")
interim_dir = os.path.join(root_dir, "data", "interim")

# software directory
ranch_dir = os.path.join("D:/github/Ranch")

# the folder where SharedStreets extractions live

shst_extract_dir = os.path.join(external_dir, "sharedstreets_extract")

# the folder where OSMNX extractions live

osm_extract_dir = os.path.join(external_dir, "osmnx_extract")
parameters = Parameters(ranch_base_dir = ranch_dir)
parameters.standard_crs
for d in [root_dir, external_dir, interim_dir, shst_extract_dir, osm_extract_dir]:
    if os.path.isdir(d):
        print(f"{d} exists!")
        continue
    
    print(f"creating {d}")
    os.makedirs(d)<jupyter_output>D:/merced exists!
D:/merced\data\external exists!
D:/merced\data\interim exists!
D:/merced\data\external\sharedstreets_extract exists!
D:/merced\data\external\osmnx_extract exists!<jupyter_text>Step 1 - SharedStreets ExtractionExtracts sharedstreets representation of OSM links.The input for this step is the polygon boundary file for the area.The output for this step is geojson files from SharedStreets API.<jupyter_code># the polygon file for the area

input_polygon_file = os.path.join(
    external_dir,
    "sharedstreets_extract", 
    "merced.shp"
)
# the SharedStreets extraction file is saved to the 'output_dir' argument

ranch.run_shst_extraction(
    input_polygon_file = input_polygon_file,
    output_dir = shst_extract_dir
)<jupyter_output>epsg:4326
2022-01-07 16:52:28, INFO: Exporting boundry file D:/merced\data\external\sharedstreets_extract\boundary.0.geojson
2022-01-07 16:52:28, INFO: extracting for polygon 0<jupyter_text>Step 2 - OSMNX ExtractionExtracts complete OSM attributes using OSMNX.The input for this step is the polygon boundary file for the area.The output for this step is geojson files from OSMNX.<jupyter_code># the OSMNX extraction file is saved to the 'output_dir' argument

ranch.run_osmnx_extraction(
    input_polygon_file = input_polygon_file,
    output_dir = os.path.join(external_dir, "osmnx_extract")
)<jupyter_output><empty_output><jupyter_text>Step 3 - Consolidate SharedStreets and OSMNX extractionsconsolidates raw roadway data and create initial roadway networks<jupyter_code># build and returns roaday network object from extraction files

roadway_network = Roadway.create_roadway_network_from_extracts(
    shst_extract_dir = shst_extract_dir,
    osm_extract_dir = osm_extract_dir,
    parameters = parameters,
)
RanchLogger.info("Initial network has {} links".format(roadway_network.links_df.shape[0]))
RanchLogger.info("Initial network has {} nodes".format(roadway_network.nodes_df.shape[0]))
RanchLogger.info("Initial network has {} shapes".format(roadway_network.shapes_df.shape[0]))<jupyter_output>2022-01-07 17:01:39, INFO: Initial network has 102371 links
2022-01-07 17:01:39, INFO: Initial network has 38693 nodes
2022-01-07 17:01:39, INFO: Initial network has 52441 shapes<jupyter_text>Optional: If the user wants to write out the roadway network in standard format, they can do the following, otherwise no need to write out<jupyter_code>RanchLogger.info("write out shape geojson")

shape_prop = ['id', 'fromIntersectionId', 'toIntersectionId', 'forwardReferenceId', 'backReferenceId']
shape_geojson = link_df_to_geojson(
    roadway_network.shapes_df, 
    shape_prop
)

with open(os.path.join(interim_dir,"step3_shapes.geojson"), "w") as f:
    json.dump(shape_geojson, f)

RanchLogger.info("write out node geojson")

node_prop = roadway_network.nodes_df.drop('geometry', axis = 1).columns.tolist()
node_geojson = point_df_to_geojson(
    roadway_network.nodes_df, 
    node_prop
)

with open(os.path.join(interim_dir,"step3_nodes.geojson"), "w") as f:
    json.dump(node_geojson, f)

RanchLogger.info("write out link json")

link_prop = roadway_network.links_df.drop(
    ['geometry'], 
    axis = 1
).columns.tolist()

out = roadway_network.links_df[link_prop].to_json(orient = "records")

with open(os.path.join(interim_dir,"step3_links.json"), 'w') as f:
    f.write(out)
    
# the standard format for links are links.json, without the geometry
# here writing out links in links.geojson in case the user wants to visualize links

RanchLogger.info("write out link geojson")

link_prop = roadway_network.links_df.drop('geometry', axis = 1).columns.tolist()
link_geojson = link_df_to_geojson(
    roadway_network.links_df, 
    link_prop
)

with open(os.path.join(interim_dir,"step3_links.geojson"), "w") as f:
    json.dump(link_geojson, f)<jupyter_output>2022-01-04 23:12:51, INFO: write out shape geojson
2022-01-04 23:13:09, INFO: write out node geojson
2022-01-04 23:13:21, INFO: write out link json
2022-01-04 23:13:25, INFO: write out link geojson<jupyter_text>Optional: it's more efficient if the user writes out the intermediate network object in pickle<jupyter_code>working_network_filename = os.path.join(interim_dir,"step3_network.pickle")
pickle.dump(roadway_network, open(working_network_filename, 'wb'))<jupyter_output><empty_output><jupyter_text>Step 4 - Third-Party Roadway Data ConflationConflates e.g. county network with roadway network using SharedStreets.The input for this step is the 3rd-party geodatabases to conflate.The output of this step is the SharedStreets conflation outputs in geojson.<jupyter_code># define the input 3rd-party file, either shapefile or geojson

input_network_file = os.path.join(external_dir, "sjmodel", "Network", "2015","TCM_MASTER_2A22_102717.shp")
# if the CRS is undefined in the input network file, user needs to specify:

input_crs = CRS("ESRI:102643")
# the output is saved to the 'output_dir' argument
# user can specify the match option (reference here) 
# by 'custom_match_option'
# if not specified, use the default
# match option would vary by the input network

ranch.run_shst_match(
    input_network_file = input_network_file,
    input_crs = input_crs,
    output_dir = os.path.join(external_dir, "sjmodel", "shst_match"),
    custom_match_option = '--tile-hierarchy=8 --search-radius=50 --snap-intersections'
)<jupyter_output>2022-01-04 16:18:56, INFO: input network D:/sanjoaquin\data\external\sjmodel\Network\2015\TCM_MASTER_2A22_102717.shp has crs : None
2022-01-04 16:18:58, INFO: Input network for shst match does not have unique IDs, generating unique IDs
2022-01-04 16:18:58, INFO: Generated 49910 unique IDs for 49910 links in the input network
2022-01-04 16:18:58, INFO: Exporting shst match input - ID-ed geometry file D:/sanjoaquin\data\external\sjmodel\shst_match\TCM_MASTER_2A22_102717.geojson
2022-01-04 16:19:02, INFO: Exporting ID-ed network file D:/sanjoaquin\data\external\sjmodel\shst_match\TCM_MASTER_2A22_102717.full.geojson<jupyter_text>Step 5 - Tidy Up RoadwayIdentifies drive dead-ends, cul-de-secs, numbering links and nodes, etc.The input for this step is the polygon file with subregion identifier, e.g. county.This step labels each link and node with the county name, and assign model IDs for links and nodes.<jupyter_code># this is the input county polygon file with county names

county_boundary_file = os.path.join(
    external_dir,
    "sharedstreets_extract", 
    "merced.shp"
)

# specify the column to look for county name

county_variable_name = 'NAME'
roadway_network.tidy_roadway(
    county_boundary_file = county_boundary_file,
    county_variable_name = county_variable_name,
    create_node_link_id = False
)<jupyter_output>2022-01-07 17:10:09, INFO: Starting Step 5 Tidy Roadway
2022-01-07 17:10:09, INFO: Joining network with county boundary file for ['Merced'] county
2022-01-07 17:10:34, INFO: Dropping links and nodes that are outside of ['Merced'] county
2022-01-07 17:10:34, INFO: Droppping circular links
2022-01-07 17:10:34, INFO: Flagging dead-end streets for drive network
2022-01-07 17:10:35, INFO: Making dead-end streets drive_access = 0
2022-01-07 17:10:37, INFO: Dropping alternative links between same AB nodes<jupyter_text>Step 6 - Build Transit Network from GTFSBuild standard transit network from GTFS<jupyter_code># the directory where all gtfs feeds are located
# e.g. gtfs_dir/sjrtd_2015_0127/routes.txt
# e.g. gtfs_dir/bart_2015_0127/routes.txt

gtfs_dir = os.path.join(external_dir, "gtfs", "2021")
# read gtfs into transit object

transit_network = Transit.load_all_gtfs_feeds(
    gtfs_dir = gtfs_dir,
    roadway_network= roadway_network,
    parameters=parameters
)
# main activities - routing buses, creating rails

transit_network.build_standard_transit_network(
    multithread_shst_match = True,
    multithread_shortest_path = False
)
# check if shortest path transit builder failed any transit shapes

transit_network.shortest_path_failed_shape_list
transit_network.write_standard_transit(
    path = interim_dir
)<jupyter_output><empty_output><jupyter_text>Step 7 - Build Centroid Connectorsbuilds centroid connectors from TAZ, MAZ<jupyter_code>taz_polygon_file = os.path.join(
    external_dir,
    "taz",
    "SJ_TAZ_Aug2011.shp"
)
roadway_network.build_centroid_connectors(
    build_taz_drive = True,
    build_taz_active_modes = True,
    input_taz_polygon_file = taz_polygon_file
)<jupyter_output><empty_output><jupyter_text>Step 8 - Write out Standard Format<jupyter_code>roadway_network.standard_format(
    county_boundary_file = os.path.join(external_dir,
        "sharedstreets_extract", 
        "merced.shp"),
    county_variable_name = 'NAME'
)
RanchLogger.info("write out shape geojson")

shape_prop = ['id', 'fromIntersectionId', 'toIntersectionId', 'forwardReferenceId', 'backReferenceId']
shape_geojson = link_df_to_geojson(
    roadway_network.shapes_df, 
    shape_prop
)

with open(os.path.join(interim_dir,"step8_shapes.geojson"), "w") as f:
    json.dump(shape_geojson, f)

RanchLogger.info("write out node geojson")

node_prop = roadway_network.nodes_df.drop('geometry', axis = 1).columns.tolist()
node_geojson = point_df_to_geojson(
    roadway_network.nodes_df, 
    node_prop
)

with open(os.path.join(interim_dir,"step8_nodes.geojson"), "w") as f:
    json.dump(node_geojson, f)

RanchLogger.info("write out link json")

link_prop = roadway_network.links_df.drop(
    ['geometry'], 
    axis = 1
).columns.tolist()

out = roadway_network.links_df[link_prop].to_json(orient = "records")

with open(os.path.join(interim_dir,"step8_links.json"), 'w') as f:
    f.write(out)
    
# the standard format for links are links.json, without the geometry
# here writing out links in links.geojson in case the user wants to visualize links

RanchLogger.info("write out link geojson")

link_prop = roadway_network.links_df.drop('geometry', axis = 1).columns.tolist()
link_geojson = link_df_to_geojson(
    roadway_network.links_df, 
    link_prop
)

with open(os.path.join(interim_dir,"step8_links.geojson"), "w") as f:
    json.dump(link_geojson, f)<jupyter_output>2022-01-07 18:29:15, INFO: write out shape geojson
2022-01-07 18:29:20, INFO: write out node geojson
2022-01-07 18:29:23, INFO: write out link json
2022-01-07 18:29:24, INFO: write out link geojson
{'fraction_non_alphanumeric': 0.09366531165311653, 'fraction_numerical': 0.041497289972899726, 'mean_word_length': 5.56420233463035, 'pattern_counts': {'":': 0, '<': 42, '<?xml version=': 0, '>': 42, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '12', 'dir': 'jupyter-structured-clean-dedup', 'id': '618728', 'license': 'BSD-3-Clause', 'n_tokens_mistral': 4407, 'n_tokens_neox': 4005, 'n_words': 1040, 'path': 'notebooks/Ranch-Demo.ipynb', 'repo_name': 'wsp-sag/Ranch', 'size': '11808'}