<jupyter_start><jupyter_text>This notebook documents how to query traffic-aware travel times, using the Mapbox Directions API.  Before running this you should have a cleaned graph object ready to use.  TODO: Clean this script, and modularize this processs  Load GOSTnets library and graph<jupyter_code>import os, sys, math, time

# GostNetsPath = r'C:\Users\gost_\Desktop\lima\GOSTNets\GOSTNets'
# GostNetsPath = '/Users/andreschamorro/Repos/GOST_PublicGoods/GOSTNets/GOSTNets'

print("Current directory is %s" % os.getcwd())
GostNetsPath = os.path.expanduser("~/Repos/GOST_PublicGoods/GOSTNets/GOSTNets")
sys.path.insert(0, GostNetsPath)
# print(sys.path)

import GOSTnet as gn
# import OSMParser
import LoadOSM as losm

import networkx as nx
import geopandas as gpd
import numpy as np
import osmnx as ox
import pandas as pd
import rasterio
import matplotlib.pyplot as plt
import inspect
os.chdir(os.path.expanduser("~/Data/Dhaka"))
!ls
G = nx.read_gpickle("dhaka_clean_small.pickle")
print(nx.info(G))
gn.example_edge(G,1)
edges = gn.edge_gdf_from_graph(G)
edges.infra_type.value_counts()<jupyter_output><empty_output><jupyter_text>Filter roads of interest Check the Mapbox telemetry documentation (https://www.mapbox.com/telemetry/) to get a sense of the quality of traffic data for your area of interest, and which roads have data. If you wanted to inspect this map in QGIS/ArcMap you can make a new Mapbox Style using the navigation template (which contains traffic data), and then load it as a WMTS service. For example, this is is the coverage for Dhaka.  ![Example_Dhaka.JPG](attachment:Example_Dhaka.JPG) In this case we select only trunk, primary, secondary, and tertiray roads.<jupyter_code>accepted_road_types = ['tertiary','secondary','primary','trunk','primary_link','trunk_link','tertiary_link','secondary_link']
edges = edges[edges.infra_type.isin(accepted_road_types)]
len(set(edges.id))
len(edges)<jupyter_output><empty_output><jupyter_text>Note: Edges don't have a unique id.<jupyter_code>for idx, row in edges.iterrows():
    edges.loc[idx, 'start_x'] = G.nodes[row.stnode]['x']
    edges.loc[idx, 'start_y'] = G.nodes[row.stnode]['y']
    edges.loc[idx, 'end_x'] = G.nodes[row.endnode]['x']
    edges.loc[idx, 'end_y'] = G.nodes[row.endnode]['y']    
edges.head()<jupyter_output><empty_output><jupyter_text>Build and make requests<jupyter_code>## library Imports
import json, sys, os, time, argparse, logging
import shapely

import pandas as pd
import numpy as np
try:
    import urllib.request as url   # python 3
except:
    import urllib2 as url  # python 2
mb_token = '--ENTER YOUR MAPBOX TOKEN HERE--'
base_url = 'https://api.mapbox.com/directions/v5/mapbox/driving-traffic/'
end_url = f'?&{mb_token}'
numcalls = 0
loop_count = 1
new_df = pd.DataFrame(columns=['start_node', 'end_node', 'duration'], index=edges.index)
start = time.time()
for idx, row in edges.iterrows():
    
    # build request
    start_x = row['start_x']
    start_y = row['start_y']
    end_x = row['end_x']
    end_y = row['end_y']
    coordinates = str(start_x)+','+str(start_y)+';'+str(end_x)+','+str(end_y)
    request = base_url+coordinates+end_url
    r = url.urlopen(request)
    data = json.loads(r.read().decode('utf-8'))['routes'][0]['duration']

    # assign response duration value to new df
    new_df.loc[idx,'start_node'] = row['stnode']
    new_df.loc[idx,'end_node'] = row['endnode']
    new_df.loc[idx,'duration'] = data
    
    numcalls += 1
    if numcalls == 299:
        
        elapsed_seconds = (time.time() - start)%60
        print(f"Did {numcalls+1} calls in {elapsed_seconds:.2f} seconds, now wait {60-elapsed_seconds:.2f}, {(300*loop_count)/len(edges):.2%} complete")
        new_df.to_csv('new_edges_temp.csv')
        time.sleep(60-elapsed_seconds)

        # reset count
        numcalls = 0
        start = time.time()
        loop_count += 1<jupyter_output>Did 300 calls in 42.64 seconds, now wait 17.36, 2.72% complete
Did 300 calls in 39.71 seconds, now wait 20.29, 5.44% complete
Did 300 calls in 38.38 seconds, now wait 21.62, 8.16% complete
Did 300 calls in 39.43 seconds, now wait 20.57, 10.88% complete
Did 300 calls in 42.74 seconds, now wait 17.26, 13.60% complete
Did 300 calls in 41.66 seconds, now wait 18.34, 16.32% complete
Did 300 calls in 41.19 seconds, now wait 18.81, 19.04% complete
Did 300 calls in 41.91 seconds, now wait 18.09, 21.76% complete
Did 300 calls in 40.50 seconds, now wait 19.50, 24.48% complete
Did 300 calls in 41.89 seconds, now wait 18.11, 27.20% complete
Did 300 calls in 43.01 seconds, now wait 16.99, 29.92% complete
Did 300 calls in 33.78 seconds, now wait 26.22, 32.64% complete
Did 300 calls in 42.15 seconds, now wait 17.85, 35.36% complete
Did 300 calls in 42.52 seconds, now wait 17.48, 38.08% complete
Did 300 calls in 44.08 seconds, now wait 15.92, 40.81% complete
Did 300 calls in 41.51 seconds, now wait 18[...]<jupyter_text>re start process, catching errors where duration wasn't available<jupyter_code>edges_failed = edges.loc[new_df[new_df.duration.isna()].index]
start = time.time()
for idx, row in edges_failed.iterrows():
    
    # build request
    start_x = row['start_x']
    start_y = row['start_y']
    end_x = row['end_x']
    end_y = row['end_y']
    coordinates = str(start_x)+','+str(start_y)+';'+str(end_x)+','+str(end_y)
    request = base_url+coordinates+end_url
    r = url.urlopen(request)
    try:
        data = json.loads(r.read().decode('utf-8'))['routes'][0]['duration']
    except:
        data = np.nan
        print(f'failed at idx {idx}')

    # assign response duration value to new df
    new_df.loc[idx,'start_node'] = row['stnode']
    new_df.loc[idx,'end_node'] = row['endnode']
    new_df.loc[idx,'duration'] = data
    
    numcalls += 1
    if numcalls == 299:
        
        elapsed_seconds = (time.time() - start)%60
        print(f"Did {numcalls+1} calls in {elapsed_seconds:.2f} seconds, now wait {60-elapsed_seconds:.2f}, {(300*loop_count)/len(edges):.2%} complete")
        new_df.to_csv('new_edges_temp.csv')
        time.sleep(60-elapsed_seconds)

        # reset count
        numcalls = 0
        start = time.time()
        loop_count += 1
edges_join = edges.join(new_df.duration)
edges_join.head()
edges_join.to_csv("edges_duration.csv")<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.13037272006344172, 'fraction_numerical': 0.04456780333068993, 'mean_word_length': 4.181594083812654, 'pattern_counts': {'":': 0, '<': 20, '<?xml version=': 0, '>': 20, 'https://': 2, 'lorem ipsum': 0, 'www.': 1, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '5', 'dir': 'jupyter-structured-clean-dedup', 'id': '609026', 'license': 'MIT', 'n_tokens_mistral': 2440, 'n_tokens_neox': 2210, 'n_words': 599, 'path': 'Implementations/FY20/ACC_Dhaka - Access to Jobs/2. Get-Traffic-Times.ipynb', 'repo_name': 'worldbank/GOST_PublicGoods', 'size': '6305'}