<jupyter_start><jupyter_text>Define data paths, reload config file<jupyter_code>import argparse
import yaml
import os
import subprocess
import argparse
import os
import pandas as pd
import yaml
import sys
import argparse
import osmnx as ox
import fiona
import shutil
import os
import re
import csv
import geojson
import json
import requests
import yaml
import sys
from shapely.geometry import mapping, Polygon, LineString, LinearRing
import numpy as np
pd.options.mode.chained_assignment = None
pd.set_option('display.max_columns', 500)
BASE_DIR = "C:\\Users\\Daniel\\Documents\\ML\\Transurban V2"
CURR_FP = os.path.join(BASE_DIR, 'src')
CONFIG_FP = os.path.join(BASE_DIR, 'src', 'config', 'Melbourne.yml')
config_file = CONFIG_FP
data_dir = os.path.join(BASE_DIR, 'data', 'Melbourne')
DATA_DIR = data_dir
# Define file paths and check directories exist
RAW_DIR = os.path.join(data_dir, "raw")
RAW_CRASH_DIR = os.path.join(RAW_DIR, 'crash')
PROCESSED_CRASH_DIR = os.path.join(data_dir, 'processed', 'crash')
PROCESSED_MAPPING_DIR = os.path.join(data_dir, 'processed', 'mapping')
MAP_DIR = os.path.join(DATA_DIR, 'processed/maps')
DOC_DIR = os.path.join(DATA_DIR, 'processed/mapping')
PROCESSED_CRASH_DIR = os.path.join(DATA_DIR, 'processed/crash')
PROCESSED_DATA_DIR = os.path.join(DATA_DIR, 'processed')
CRASH_DIR = os.path.join(BASE_DIR, 'data/Melbourne/processed/crash')
with open(CONFIG_FP) as f:
    config = yaml.safe_load(f)
DATA_FP = os.path.join(BASE_DIR, 'data', config['name'])
import argparse
import os
import shutil
import tzlocal
from data.util import geocode_address

BASE_DIR = "C:\\Users\\Daniel\\Documents\\ML\\Transurban V2"


def make_config_file(yml_file, timezone, city, folder, crash_file_path, map_file_path, map_inters_file_path, atmosphere_file_path, merged_file_path, cat_feat, cont_feat, keep_feat):

    address = geocode_address(city)

    f = open(yml_file, 'w')
    f.write(
        "# City name\n" +
        "city: {}\n".format(city) +
        "# City centerpoint latitude & longitude\n" +
        "city_latitude: {}\n".format(address[1]) +
        "city_longitude: {}\n".format(address[2]) +
        "# City's time zone [defaults to local to,e of computer]\n" +
        "timezone: {}\n".format(timezone) +
        "# The folder under data where this city's data is stored\n" +
        "name: {}\n".format(folder) +
        "# Limit crashes to between start and end date\n" +
        "startdate: \n" +
        "enddate: \n" +
        "#################################################################\n" +
        "crash_files:\n" +
        "  {}\n".format(crash_file_path) +
        "  {}\n".format(map_file_path) +
        "  {}\n".format(map_inters_file_path) +
        "  {}\n".format(atmosphere_file_path) +
        "cat_feat: {} \n".format(cat_feat) +
        "cont_feat: {} \n".format(cont_feat) +
        "keep_feat: {} \n".format(keep_feat) +
        "merged_data: {}".format(merged_file_path)
    )

    f.close()
    print("Wrote new configuration file in {}".format(yml_file))


def make_js_config(jsfile, city, folder):
    address = geocode_address(city)

    f = open(jsfile, 'w')
    f.write(
        'var config = {\n' +
        '    MAPBOX_TOKEN: "<KEY>",\n' +
        '    cities: [\n' +
        '        {\n' +
        '            name: "{}",\n'.format(city) +
        '            id: "{}",\n'.format(folder) +
        '            latitude: {},\n'.format(str(address[1])) +
        '            longitude: {},\n'.format(str(address[2])) +
        '        }\n' +
        '    ]\n' +
        '}\n'
    )
    f.close()


city = "Melbourne"
folder = "Melbourne"
print('Using the default raw file locations. If they do not exist, please include raw file path arguments in cmd call.')
crash_file = 'C:/Users/Daniel/Documents/ML/Transurban V2/data/raw/crash.csv'
map_file = 'C:/Users/Daniel/Documents/ML/Transurban V2/data/raw/map.csv'
map_inters_file = 'C:/Users/Daniel/Documents/ML/Transurban V2/data/raw/map_inters.csv'
atmosphere_file = 'C:/Users/Daniel/Documents/ML/Transurban V2/data/raw/atmosphere.csv'

# Get our various file paths
DATA_FP = os.path.join(BASE_DIR, 'data', folder)
PROCESSED_DIR = os.path.join(DATA_FP, 'processed')
RAW_DIR = os.path.join(DATA_FP, 'raw')
RAW_CRASH_DIR = os.path.join(RAW_DIR, 'crash')

crash_file_path = os.path.join(RAW_CRASH_DIR, 'crash.csv')
map_file_path = os.path.join(RAW_CRASH_DIR, 'map.csv')
map_inters_file_path = os.path.join(RAW_CRASH_DIR, 'map_inters.csv')
atmosphere_file_path = os.path.join(RAW_CRASH_DIR, 'atmosphere.csv')
merged_file_path = os.path.join(PROCESSED_DIR, 'canon.csv.gz')

# Define our categorical / continuous features for usage in modelling
cat_feat = ['HOUR', 'DAY_OF_WEEK', 'MONTH', 'DEGREE_URBAN', 'LIGHT_COND', 'ATMOSPH_COND', 'NODE_TYPE_INT',
            'COMPLEX_INT', 'hwy_type', 'inter', 'intersection_segments', 'lanes', 'oneway', 'signal', 'streets', 'direction']

cont_feat = ['SPEED_ZONE', 'osm_speed', 'LAST_7_DAYS', 'LAST_30_DAYS', 'LAST_365_DAYS', 'LAST_1825_DAYS', 'LAST_3650_DAYS']

# Define our features to keep until the last step where we strip down to modelling features
keep_feat = cat_feat + cont_feat + ['display_name', 'intersection', 'segment_id']

# Create our data paths
if not os.path.exists(DATA_FP):
    print("Making directory structure under " + DATA_FP)
    os.makedirs(DATA_FP)
    os.makedirs(os.path.join(DATA_FP, 'raw'))
    os.makedirs(os.path.join(DATA_FP, 'processed'))
    os.makedirs(os.path.join(DATA_FP, 'standardized'))
    os.makedirs(RAW_CRASH_DIR)

    # We copy across all raw data files.
    # Note: Merged is not copied because not yet created.
    shutil.copyfile(crash_file, crash_file_path)
    shutil.copyfile(map_file, map_file_path)
    shutil.copyfile(map_inters_file, map_inters_file_path)
    shutil.copyfile(atmosphere_file, atmosphere_file_path)
else:
    print(folder + "folder already initialized, skipping")

# Create our yml config file
yml_file = os.path.join(BASE_DIR, 'src/config/' + folder + '.yml')
if not os.path.exists(yml_file):
    make_config_file(yml_file, tzlocal.get_localzone().zone, city, folder, crash_file_path, map_file_path, map_inters_file_path, atmosphere_file_path, merged_file_path, cat_feat, cont_feat, keep_feat)

# Create our js config file
reports_file_path = os.path.join(BASE_DIR, 'reports', folder)
print(reports_file_path)
if not os.path.exists(reports_file_path):
    print('Making reports file path')
    os.makedirs(reports_file_path)

js_file_path = os.path.join(BASE_DIR, 'reports', folder, 'config.js')
if not os.path.exists(js_file_path):
    print("Writing config.js")
    make_js_config(js_file_path, city, folder)<jupyter_output><empty_output><jupyter_text>Pipeline  Standardize Crashes<jupyter_code>def read_clean_combine_crash(RAW_CRASH_DIR):
    # Get file names and read in csv's
    crash_file = os.path.join(RAW_CRASH_DIR, 'crash.csv')
    map_file = os.path.join(RAW_CRASH_DIR, 'map.csv')
    map_inters_file = os.path.join(RAW_CRASH_DIR, 'map_inters.csv')
    atmosphere_file = os.path.join(RAW_CRASH_DIR, 'atmosphere.csv')

    crash_df = pd.read_csv(crash_file)
    map_df = pd.read_csv(map_file)
    map_inters_df = pd.read_csv(map_inters_file)
    atmosphere_df = pd.read_csv(atmosphere_file)

    # crash_df: drop unwanted columns and establish mappings
    crash_df_cols_reduced = ['ACCIDENT_NO', 'ACCIDENTDATE', 'ACCIDENTTIME', 'DAY_OF_WEEK', 'LIGHT_CONDITION', 'NODE_ID', 'ROAD_GEOMETRY', 'SPEED_ZONE']
    geom_mapping_cols = ['ROAD_GEOMETRY', 'Road Geometry Desc']
    accident_type_mapping_cols = ['ACCIDENT_TYPE', 'Accident Type Desc']
    DCA_code_mapping_cols = ['DCA_CODE', 'DCA Description']
    light_condition_mapping_cols = ['LIGHT_CONDITION', 'Light Condition Desc']

    geom_mapping = crash_df[geom_mapping_cols]
    geom_mapping = geom_mapping.drop_duplicates().sort_values(by='ROAD_GEOMETRY').reset_index(drop=True)

    accident_type_mapping = crash_df[accident_type_mapping_cols]
    accident_type_mapping = accident_type_mapping.drop_duplicates().sort_values(by='ACCIDENT_TYPE').reset_index(drop=True)

    light_condition_mapping = crash_df[light_condition_mapping_cols]
    light_condition_mapping = light_condition_mapping.drop_duplicates().sort_values(by='LIGHT_CONDITION').reset_index(drop=True)

    DCA_code_mapping = crash_df[DCA_code_mapping_cols]
    DCA_code_mapping = DCA_code_mapping.drop_duplicates().sort_values(by="DCA_CODE").reset_index(drop=True)

    crash_df_reduced = crash_df[crash_df_cols_reduced]

    # Map_df: drop unwanted columns, create node type mapping
    node_type_mapping = pd.DataFrame({'NODE_TYPE_INT': [0, 1, 2, 3], 'NODE_TYPE': ['I', 'N', 'O', 'U'], 'NODE_DESC': ['Intersection', 'Non-intersection', 'Off-road', 'Unknown']})

    map_df['NODE_TYPE_INT'] = ""
    for index, row in node_type_mapping.iterrows():
        map_df['NODE_TYPE_INT'].loc[map_df['NODE_TYPE'] == row['NODE_TYPE']] = row['NODE_TYPE_INT']

    map_df_reduced_cols = ['ACCIDENT_NO', 'NODE_ID', 'NODE_TYPE_INT', 'LGA_NAME', 'Deg Urban Name', 'Lat', 'Long']
    map_df_reduced = map_df[map_df_reduced_cols]

    # map_iters_df: drop unwanted columns, creat node-to-complex node mapping
    complex_node_mapping_cols = ['NODE_ID', 'COMPLEX_INT_NO']
    complex_node_mapping = map_inters_df[complex_node_mapping_cols]
    complex_node_mapping = complex_node_mapping.drop_duplicates().sort_values(by="NODE_ID").reset_index(drop=True)

    map_inters_df_reduced = map_inters_df[['ACCIDENT_NO', 'COMPLEX_INT_NO']]

    # atmosphere_df: drop unwanted columns, create atmosphere mapping
    atmosphere_mapping_cols = ['ATMOSPH_COND', 'Atmosph Cond Desc']
    atmosphere_mapping = atmosphere_df[atmosphere_mapping_cols]
    atmosphere_mapping = atmosphere_mapping.drop_duplicates().sort_values(by="ATMOSPH_COND").reset_index(drop=True)

    atmosphere_df_reduced_cols = ['ACCIDENT_NO', 'ATMOSPH_COND']
    atmosphere_df_reduced = atmosphere_df[atmosphere_df_reduced_cols]

    # Drop duplicates from all dataframes
    # Note: most of these duplicates are legitimate
    # Chain effects of a crash are given different incident numbers. We will treat it as one crash however.
    crash_df_reduced.drop_duplicates(subset="ACCIDENT_NO", inplace=True)
    map_df_reduced.drop_duplicates(subset="ACCIDENT_NO", inplace=True)
    map_inters_df_reduced.drop_duplicates(subset="ACCIDENT_NO", inplace=True)
    atmosphere_df_reduced.drop_duplicates(subset="ACCIDENT_NO", inplace=True)

    # Begin joining dataframes on 'ACCIDENT_NO'.
    # Joining by 'outer', means that if some accident numbers are in one DF but not in another, the accident will still be recorded but will be missing columns
    # The validate option ensures that when merging, each DF only has one instance of each accident number
    crashes_and_atmos = pd.merge(crash_df_reduced, atmosphere_df_reduced, on='ACCIDENT_NO', how='outer', validate='one_to_one')
    inters_and_complex = pd.merge(map_df_reduced, map_inters_df_reduced, on='ACCIDENT_NO', how='outer', validate='one_to_one')
    crashes_df = pd.merge(crashes_and_atmos, inters_and_complex, on='ACCIDENT_NO', how='outer', validate='one_to_one')

    # Many NA's within the 'COMPLEX_INT_NO' column. Fill some of these.
    # Some NA's associated with strange NODE_ID's from original crash DF, meaning we couldn't map to Lat / Lon. Remove these.
    # Drop the extra 'NODE_ID' column we have gained
    crashes_df['COMPLEX_INT_NO'] = crashes_df['COMPLEX_INT_NO'].fillna(0)
    crashes_df.dropna(subset=['Lat', 'Long'], inplace=True)
    crashes_df.drop(['NODE_ID_y'], axis=1, inplace=True)

    # Make sure there are no more NA values left anywhere within DF
    if len(crashes_df[crashes_df.isna().any(axis=1)]) != 0:
        print('There are still NA values left within crashes_df during standardization')
        print('Please check this manually. Exiting.')
        sys.exit(1)

    # Standardize column names
    crashes_df.rename(columns={"NODE_ID_x": "NODE_ID", "ACCIDENTDATE": "ACCIDENT_DATE", "ACCIDENTTIME": "ACCIDENT_TIME",
                               "LGA_NAME": "SUBURB", "Deg Urban Name": "DEGREE_URBAN", "Lat": "LAT", "Long": "LON", "LIGHT_CONDITION": "LIGHT_COND"}, inplace=True)

    # Change dtype of columns to integers [these were changed due to the merge step creating NA values]
    crashes_df[['NODE_ID', 'COMPLEX_INT_NO']] = crashes_df[['NODE_ID', 'COMPLEX_INT_NO']].astype(int)

    # Feature engineering
    # Add a binary value for if a crash occured at a complex node
    # Add values for Hour / Month as seasonality features
    crashes_df['COMPLEX_NODE'] = 0
    crashes_df['COMPLEX_NODE'].loc[crashes_df['COMPLEX_INT_NO'] > 0] = 1
    crashes_df['DATE_TIME'] = pd.to_datetime(crashes_df['ACCIDENT_DATE'] + " " + crashes_df['ACCIDENT_TIME'], format="%d/%m/%Y %H.%M.%S")
    crashes_df['HOUR'] = crashes_df.DATE_TIME.dt.hour
    crashes_df['MONTH'] = crashes_df.DATE_TIME.dt.month

    # Once again drop unwanted columns
    crashes_df.drop(['ACCIDENT_DATE', 'ACCIDENT_TIME', 'COMPLEX_INT_NO'], axis=1, inplace=True)

    # Reorder columns for easier reading
    crashes_df = crashes_df[['ACCIDENT_NO', 'DATE_TIME', 'MONTH', 'HOUR', 'DAY_OF_WEEK', 'LAT', 'LON', 'SUBURB', 'NODE_ID', 'NODE_TYPE_INT', 'COMPLEX_NODE', 'LIGHT_COND', 'ATMOSPH_COND', 'SPEED_ZONE', 'ROAD_GEOMETRY', 'DEGREE_URBAN']]

    # Set 'ACCIDENT_NO' to be the index
    crashes_df.set_index('ACCIDENT_NO', inplace=True)

    # Put various mappings into a tuple to allow for easier transport
    mappings = (geom_mapping, accident_type_mapping, DCA_code_mapping, light_condition_mapping, node_type_mapping, atmosphere_mapping)

    return crashes_df, mappings


def output_crash_csv(PROCESSED_CRASH_DIR, PROCESSED_MAPPING_DIR, crashes_df, mappings):

    geom_mapping, accident_type_mapping, DCA_code_mapping, light_condition_mapping, node_type_mapping, atmosphere_mapping = mappings

    # Output resulting crashes_df and all mappings
    if not os.path.exists(PROCESSED_CRASH_DIR):
        os.makedirs(PROCESSED_CRASH_DIR)

    if not os.path.exists(PROCESSED_MAPPING_DIR):
        os.makedirs(PROCESSED_MAPPING_DIR)

    crashes_path = os.path.join(PROCESSED_CRASH_DIR, 'crashes.csv')
    crashes_df.to_csv(crashes_path)

    mapping_dfs = [geom_mapping, accident_type_mapping, DCA_code_mapping,
                   light_condition_mapping, node_type_mapping, atmosphere_mapping]
    mapping_names = ['geom_mapping.csv', 'accident_type_mapping.csv', 'DCA_code_mapping.csv',
                     'light_condition_mapping.csv', 'node_type_mapping.csv', 'atmosphere_mapping.csv']

    for mapping_df, mapping_name in zip(mapping_dfs, mapping_names):
        save_path = os.path.join(PROCESSED_MAPPING_DIR, mapping_name)
        mapping_df.to_csv(save_path)


def output_crash_json(PROCESSED_CRASH_DIR, crashes_df):
    output_file = PROCESSED_CRASH_DIR + '/crashes.json'
    crashes_df.to_json(output_file, orient='index')

crashes_df, mappings = read_clean_combine_crash(RAW_CRASH_DIR)
output_crash_csv(PROCESSED_CRASH_DIR, PROCESSED_MAPPING_DIR, crashes_df, mappings)
output_crash_json(PROCESSED_CRASH_DIR, crashes_df)<jupyter_output><empty_output><jupyter_text>DATA GENERATION  OSM create maps<jupyter_code>CURR_DIR = 'C:\\Users\\Daniel\\Documents\\ML\\Transurban V2\\src\\data'
sys.path.append(CURR_DIR)

import util
def find_osm_polygon(city):
    """Interrogate the OSM nominatim API for a city polygon.

    Nominatim may not always return city matches in the most intuitive order,
    so results need to be searched for a compatible polygon. The index of the
    polygon is required for proper use of osmnx.graph_from_place(). Some cities
    do not have a polygon at all, in which case they defer to using
    graph_from_point() with city lat & lng.

    Args:
        city (str): city to search for
    Returns:
        int: index of polygon+1 (becomes the correct 'which_result' value)
        None: if no polygon found
    """

    # Old Request that I couldn't get working for greater melbourne region
    # search_params = {'format': 'json', 'limit': 5, 'dedupe': 0, 'polygon_geojson': 1, 'city': city}
    # url = 'https://nominatim.openstreetmap.org/search'
    # response = requests.get(url, params=search_params)

    # Returns the Greater Melbourne region.
    # The response object can be dug into via enumerate.
    # match['geojson'] looks like {'type' : 'Polygon', 'coordinates': [[[x, y], [x2, y2], ...]]}
    #response = requests.get('https://nominatim.openstreetmap.org/search.php?q=greater+melbourne&polygon_geojson=1&format=json')
    response = requests.get('https://nominatim.openstreetmap.org/search.php?q=melbourne&polygon_geojson=1&format=json')
    for index, match in enumerate(response.json()):

        # To be used by graph_from_place needs to be a Polygon or MultiPolygon
        if (match['geojson']['type'] in ['Polygon', 'MultiPolygon']):
            return index + 1, match['geojson']

    return None, None


def expand_polygon(polygon, points_file, verbose=False, max_percent=.1, expand_polygon_bool=False):
    """
    Read the crash data, determine what proportion of crashes fall outside
    the city polygon
    Args:
        polygon - city polygon
        points_file - json points file
        Optional: max_percent (in case you want to override the maximum
            percent that can be outside the original polygon to buffer)
    Returns:
        Updated polygon if it was a polygon to start with, None otherwise
    """

    # Only support for polygons
    if polygon['type'] != 'Polygon':
        if verbose:
            print('Within expand_polygon. Polygon\'s type is {}. We only support Polygon'.format(polygon['type']))
        return None

    # Remap polygon coordinates using get_reproject_point [default from 4326 to 3857 methods of geodesy]. Note 4326 is our normal concept of lat / lon.
    # In general, 4326 is good for accurate storage, but is 3D. 3857 is good for 2D representation, but has issues with accuracy.
    # There is some strangeness here with regards to ordering of x[1] and x[0]. This is because get_reproject_point flips it around, but didn't want to fix as might break something else.
    polygon_coords = [util.get_reproject_point(x[1], x[0], verbose, coords=True) for x in polygon['coordinates'][0]]
    if verbose:
        print('Within osm_create_maps.expand_polygon')
        print('Remapping lat / lon via utils.get_reproject_point from epsg:4326 to epsg:3857 by default')
        print('Coords originally looked like:', polygon['coordinates'][0][0][1], polygon['coordinates'][0][0][0])
        print('Transformed coords look like:', polygon_coords[0])

    # Use shapely.geometry.polygon to create polygon object from coordinates
    poly_shape = Polygon(polygon_coords)
    if verbose:
        print('Having obtained poly_coords we create polygon object which looks like:', poly_shape)

    # Saving poly_shape out for visualisation
    schema = {
        'geometry': 'Polygon',
        'properties': {'id': 'int'},
    }

    with fiona.open(os.path.join(MAP_DIR, "polygon_coords.shp"), "w", "ESRI Shapefile", schema) as output:
        output.write({
            'geometry': mapping(poly_shape),
            'properties': {'id': 123}
        })

    # Reading in records from point file [in this case]. Projects lat and lon from 4326 to 3857.
    records = util.read_records(points_file, 'crash', verbose=verbose)
    if verbose:
        print('Feeding into read_records:')
        print('points_file:', points_file)
        print('Having created our record objects here is an example:', records[0])

    # Saving our records to a point shp file so we can visualise the geolocations of the crashes
    # for record in records:

    # If expand_polygon is true, check how many points from our crash records fall outside poly_shape's boundaries.
    if expand_polygon_bool:
        outside = []
        for record in records:
            if not poly_shape.contains(record.point):
                outside.append(record.point)
        outside_rate = len(outside) / len(records)
        if verbose:
            print('# crashes: {}, # records outside {}'.format(len(records), len(outside)))

        # If too many points fell outside, buffer [enlarge] the poly_shape until less than max_percent fall.
        while outside_rate > max_percent:
            print("{}% of crashes fell outside the city polygon so we are buffering".format(int(round(outside_rate, 2) * 100)))
            poly_shape, num_out = buffer_polygon(poly_shape, outside, distMax=10000, verbose=verbose)
            outside_rate = num_out / len(records)

        # Again write the shapefile to directory so we can inspect how it has changed
        with fiona.open(os.path.join(MAP_DIR, "polygon_coords_buffered.shp"), "w", "ESRI Shapefile", schema) as output:
            output.write({
                'geometry': mapping(poly_shape),
                'properties': {'id': 123}
            })

    else:
        print('NOTE: expand_poly_bool = False')
        print('Thus we are not attempting to expand polygon. Normally would check if all crash data falls within polygon, and if not expand.')
        print('Expanding takes a long time however, so can skip if you are sure your polygon from OSM is large enough.')

    # Convert back to 4326 projection
    coords = [util.get_reproject_point(
        x[1],
        x[0],
        inproj='epsg:3857',
        outproj='epsg:4326',
        coords=True
    ) for x in poly_shape.exterior.coords]
    poly_shape_4326 = Polygon(coords)

    return poly_shape_4326


def buffer_polygon(polygon, points, distMax=10000, verbose=False):
    """
    Given a set of points outside a polygon, expand the polygon
    to include points within 250 meters
    Args:
        polygon - shapely polygon
        points - list of shapely points
    Returns:
        new polygon with buffered points added
    """
    not_close = []
    add_buffers = []
    len_points = len(points)
    counter = 0

    # Find the distance between points and exterior of city polygon
    # If they are within distMax from the exterior, include them in the polygon, otherwise leave out.
    poly_ext = LinearRing(polygon.exterior.coords)
    for point in points:
        dist = polygon.distance(point)
        if dist > distMax:
            not_close.append(point)
        else:
            point2 = poly_ext.interpolate(poly_ext.project(point))
            line = LineString([(point.x, point.y), (point2.x, point2.y)])
            buff = line.buffer(50)
            add_buffers.append(buff)
        if verbose:
            print("{} / {} - {} % buffered".format(counter, len_points, 100 * counter / len_points))
            counter += 1

    # Now that all the buffers have been discovered, we create our new polygon as a union.
    for buff in add_buffers:
        polygon = polygon.union(buff)

    # Check how many polygons were still not included and return as a percentage
    num_out = len(not_close)
    if not_close:
        print("{} crashes fell outside the buffered city polygon".format(len(not_close)))
    else:
        print("Expanded city polygon to include all crash locations")

    return polygon, num_out


def simple_get_roads(config, verbose, expand_poly=False):
    """
    Use osmnx to get a simplified version of open street maps for the city
    Writes osm_nodes and osm_ways shapefiles to MAP_DIR
    Args:
        city
    Returns:
        None
    Creates:
           osm_ways.shp - the simplified road network
           osm_nodes.shp - the intersections and dead ends
    Creates directory:
           all_nodes - containing edges and nodes directories for the unsimplified road network
    """

    # Load in polygon from request to nominatim.openstreetmap.org/search
    # Polygon takes the form {'type' : 'Polygon', 'coordinates': [[[x, y], [x2, y2], ...]]}
    print("searching OSM for " + str(config['city']) + " polygon")
    polygon_pos, polygon = find_osm_polygon(config['city'])
    if verbose:
        print('Polygon_pos variables is:', polygon_pos)
        print('Polygon type is:', type(polygon))

    # If too many crashes land outside of our city polygon, expand the polygon
    if expand_poly:
        print('Expand_poly is true. About to attempt to expand the polygon to include more crashes')
        polygon = expand_polygon(polygon, os.path.join(PROCESSED_CRASH_DIR, 'crashes.json'), verbose)
    else:
        polygon = Polygon(polygon['coordinates'][0])

    # Creates a network graph from OSM data within the spatial boundaries of the passed-in shapely polygon
    print('Creating a graph from polygon data')
    G1 = ox.graph_from_polygon(polygon, network_type='drive', simplify=False)

    # Simplify graph's topology by removing all nodes that are not intersections or dead-ends.
    # Creates an edge directly between endpoints that encapsulate them, but retain the geometry of original edges,
    # saved as attribute in new edge.
    # e.g. before a curved road would have many nodes for each part of curve. Now only a node at beginning and end, but curvature is retained.
    print('Simplifying graph by removing nodes that are not intersections or dead-ends')
    G = ox.simplify_graph(G1)

    # Label dead ends here as must be done before saving object out, after which cant use count_streets_per_node
    # Will finish cleanup of node data during the geojson write-out.
    # Add dead_end to the node properties
    print('Labelling dead ends')
    streets_per_node = ox.count_streets_per_node(G)
    for node, count in list(streets_per_node.items()):
        if count <= 1:
            G.nodes()[node]['dead_end'] = True

    # save_graph_shapefile saves graph nodes and edges as ESRI shapefile
    # Save both simplified and complex graphs. The graphs extra information can be used as features.
    # Saving outputs nodes and edges folders with nodes / edges files of the format .cpg, .dbf, .prj, .shp, .shx
    # The edges have the following properties directly from OSM: access, bridge, from, highway, junction, key, lanes, length, maxspeed, name, oneway, osmid, ref, service, to, width
    # The nodes have the following properties directly from OSM: higway, osmid. 'dead_end' is added manually above.
    print('Saving graph that has all nodes which may be later used as features')
    ox.save_graph_shapefile(G1, filename='all_nodes', folder=MAP_DIR)
    print('Saving simplified graph')
    ox.save_graph_shapefile(G, filename='temp', folder=MAP_DIR)

    # Copy the simplified files in the temp directory, moving them to the MAP_DIR directory
    # Label those files that were put in the edges directory as osm_ways.ext
    # Label those files that were put in the nodes directory as osm_nodes.ext
    tempdir = os.path.join(MAP_DIR, 'temp')

    for filename in os.listdir(os.path.join(tempdir, 'edges')):
        _, extension = filename.split('.')
        shutil.move(os.path.join(tempdir, 'edges', filename), os.path.join(MAP_DIR, 'osm_ways.' + extension))

    for filename in os.listdir(os.path.join(tempdir, 'nodes')):
        _, extension = filename.split('.')
        shutil.move(os.path.join(tempdir, 'nodes', filename), os.path.join(MAP_DIR, 'osm_nodes.' + extension))

    shutil.rmtree(tempdir)


def clean_and_write(ways_file, nodes_file, result_file, DOC_DIR, verbose):
    """
    Takes several shape files in 4326 projection, created from osmnx, reprojects them, and calls write_geojson
    Args:
        ways_file - shp file for the ways
        nodes_file - shp file for the intersection and end nodes
        all_nodes_file - shp file for ALL nodes in the road network
        result_file - file to write to
        DOC_DIR - file to write highway keys to
    Returns:
        OSM_elements.geojson [or whatever result_file name you are using]
    """
    if verbose:
        print('Within osm_create_maps.clean_and_write with the following variables')
        print('ways_file:', ways_file)
        print('nodes_file:', nodes_file)
        print('results_file:', result_file)
        print('DOC_DIR:', DOC_DIR)

    # Reads osm_ways file, cleans up features and reprojects onto 3875
    # Additionally writes mapping for highway types and saves to DOC_DIR
    # cleaned_ways include way objects which are ordered dictionaries with the following keys:
    # access, area, birdge, est_width, from, highway, junction, key, lanes, length, maxspeed, name, oneway, osmid, ref, service, to, tunnel, width, hwy_type, osm_speed, signal, width_per_lane
    cleaned_ways = clean_ways(ways_file, DOC_DIR, verbose)

    # Populate the cross streets for each node and add unique ids to the ways
    # Returns nodes [same as shape file but with the added properties of 'streets', 'intersection', 'signal'] and ways [with unique osmid-fromNodeNumber-toNodeNumber string]
    # Each node in nodes comes with the following keys: dead_end, highway, osmid, ref
    nodes = fiona.open(nodes_file)
    nodes, cleaned_ways = populate_node_features_ways_ID(cleaned_ways, nodes, verbose)

    # Append nodes to ways and output all into a result_file [in this case osm_elements.geojson]
    write_geojson(cleaned_ways, nodes, result_file)


def clean_ways(orig_file, DOC_DIR, verbose):
    """
    Reads in osm_ways file, cleans up the features, and reprojects
    results into 3857 projection. Additionally writes a key which shows the correspondence
    between highway type as a string and the resulting int feature
    Features:
        width
        lanes
        hwy_type
        osm_speed
        signal
    Args:
        orig_file: Filename for original file
        result_file: Filename for resulting file in 3857 projection
        DOC_DIR: directory to write highway keys file to
    Returns:
        a list of reprojected way lines
    """
    if verbose:
        print('Within osm_create_maps.clean_ways with the following variables')
        print('orig file:', orig_file)

    way_lines = fiona.open(orig_file)

    highway_keys = {}
    results = []

    # The way_line comes with the following properties from the shp file:
    # access, area, bridge, est_width, from, highway, junction, key, lanes,
    # length, maxspeed, name, oneway, osmid, ref, service, to, tunnel, width
    for way_line in way_lines:

        # Get speed for each way. Assign 0 if not found.
        speed = get_speed(way_line['properties']['maxspeed']) if 'maxspeed' in list(way_line['properties']) else 0

        # Get width for each way. Assign 0 if not found.
        width = get_width(way_line['properties']['width']) if 'width' in list(way_line['properties']) else 0

        # Get lanes for each way
        lanes = way_line['properties']['lanes']
        if lanes:
            lanes = max([int(x) for x in re.findall('\d', lanes)])
        else:
            lanes = 0

        # Get one-way for each way
        oneway = 0
        if way_line['properties']['oneway'] == 'True':
            oneway = 1

        # Get the approximate directionality of each way
        # get_direction returns 'N-S', 'E-W', 'NE-SW', 'NW-SE' for directions
        way_coords = way_line['geometry']['coordinates']
        way_start = way_coords[0]
        way_end = way_coords[1]
        way_path_x = (way_end[0] - way_start[0])
        way_path_y = (way_end[1] - way_start[1])
        direction = get_direction(way_path_x, way_path_y)

        # Derive highway mapping
        if way_line['properties']['highway'] not in list(highway_keys.keys()):
            highway_keys[way_line['properties']['highway']] = len(highway_keys)

        # Derive width per lane
        width_per_lane = 0
        if lanes and width:
            width_per_lane = round(width / lanes)

        # Update properties with newly found variables
        # Note we include 'signal':0, but will properly update this varible later in write_geojson
        way_line['properties'].update({
            'width': width,
            'lanes': int(lanes),
            'hwy_type': highway_keys[way_line['properties']['highway']],
            'osm_speed': speed,
            'signal': 0,
            'oneway': oneway,
            'width_per_lane': width_per_lane,
            'direction': direction
        })

        results.append(way_line)

    # Write the highway keys to a mapping document in processed/mapping
    write_highway_keys(DOC_DIR, highway_keys)

    # Returns our results, which is an ordered dict of ways with the following properties:
    # access, area, birdge, est_width, from, highway, junction, key, lanes, length, maxspeed
    # name, oneway, osmid, ref, service, to, tunnel, width, hwy_type, osm_speed, signal, width_per_lane
    return results


def populate_node_features_ways_ID(ways, nodes, verbose):
    """
    For each node the 'streets' [incoming street names], 'intersection' [dead-end or intersection 0 / 1] and 'signal' [traffic lights or not 0 / 1] properties
    For each way create a unique ID based on the nodes it spans and the way ID

    Args:
        ways - a list of geojson linestrings
        nodes - a list of geojson points
    Returns:
        nodes - a dict containing the roads connected to each node
        ways - the ways, with a unique osmid-fromNodeID-toNodeID string
    """

    # node_info will hold a key for each node, and record any roads connected to the node.
    node_info = {}
    for way in ways:

        # There are some collector roads and others that don't have names. Skip these.
        # For all the others, we fill out our node_info dictionary so that for a given node it lists all the incoming road names
        if way['properties']['name']:

            # While we are still merging segments with different names, just use both roads. Revisit.
            # E.g. [Munz Avenue, Telfast Street] -> 'Munz Avenue/Telfast Street'
            if '[' in way['properties']['name']:
                way['properties']['name'] = re.sub(r'[^\s\w,]|_', '', way['properties']['name'])
                way['properties']['name'] = "/".join(way['properties']['name'].split(', '))

            # Check if the 'from' property is in the node_info keys. If it's not, create an entry for that node number.
            # Append to the node number the name of the street that is attached
            if way['properties']['from'] not in node_info.keys():
                node_info[way['properties']['from']] = []
            node_info[way['properties']['from']].append(way['properties']['name'])

            # Check if the 'to' property is in the node_info keys. If it's not, create an entry for that node number.
            # Append to the node number the name of the street it is going to
            if way['properties']['to'] not in node_info.keys():
                node_info[way['properties']['to']] = []
            node_info[way['properties']['to']].append(way['properties']['name'])

        # Add a unique identifier to way['segment_id'] which takes the form OSMID - NodeNumberFrom - NodeNumberTo
        ident = str(way['properties']['osmid']) + '-' + str(way['properties']['from']) + '-' + str(way['properties']['to'])
        way['properties']['segment_id'] = ident

    # Add the 'streets', 'intersection' and 'signal' properties to our nodes
    # Immediately previously filled out node_info to detail the incoming roads for each node
    # Now append this extra data into our nodes object, creating nodes_extra_features
    # The inbuilt 'set' function is used to eliminate duplicates
    nodes_extra_features = []
    for node in nodes:

        # Adding 'streets' property
        if node['properties']['osmid'] in node_info:
            node['properties']['streets'] = ', '.join(set(node_info[node['properties']['osmid']]))
        else:
            node['properties']['streets'] = ''

        # Adding 'dead_end' property
        if not node['properties']['dead_end']:
            node['properties']['intersection'] = 1

        # Adding 'signal' property
        if node['properties']['highway'] == 'traffic_signals':
            node['properties']['signal'] = 1

        nodes_extra_features.append(node)

    return nodes_extra_features, ways


def write_geojson(way_results, node_results, outfp):
    """
    Given a list of ways, intersection nodes, and all nodes, write them out to a geojson file.
    """

    # Get all the ways
    feats = way_results

    # Get all the nodes. Check if they have traffic lights / are a dead end and edit properties
    # Add the nodes to the feats, so that it contains nodes and ways
    for node in node_results:
        feats.append(geojson.Feature(geometry=geojson.Point(node['geometry']['coordinates']), properties=node['properties']))

    # Write out features.
    feat_collection = geojson.FeatureCollection(feats)
    with open(outfp, 'w') as outfile:
        geojson.dump(feat_collection, outfile)


def write_highway_keys(DOC_DIR, highway_keys):
    """
    Since we're creating a numeric highway key, we'd like to know what
    the numbers correspond to, so write to file the mapping from key
    to open street map highway type
    Args:
        DOC_DIR - the directory to write the file
        highway_keys - a dict associating key with string type
    """
    # Write highway keys to docs if needed for reference
    if not os.path.exists(DOC_DIR):
        os.makedirs(DOC_DIR)
    with open(os.path.join(DOC_DIR, 'highway_keys.csv'), 'w') as f:
        w = csv.writer(f)
        w.writerow(['type', 'value'])
        for item in highway_keys.items():
            w.writerow(item)


def get_width(width):
    """
    Parse the width from the openstreetmap width property field
    Args:
        width - a string
    Returns:
        width - an int
    """

    # This indicates two segments combined together.
    # For now, we just skip combined segments with different widths
    if not width or ';' in width or '[' in width:
        width = 0
    else:
        # Sometimes there's bad (non-numeric) width
        # so remove anything that isn't a number or .
        # Skip those that don't have some number in them
        width = re.sub('[^0-9\.]+', '', width)
        if width:
            width = round(float(width))
        else:
            width = 0
    return width


def get_speed(speed):
    """
    Parse the speed from the openstreetmap maxspeed property field
    If there's more than one speed (from merged ways), use the highest speed
    Args:
        speed - a string
    Returns:
        speed - an int
    """
    if speed:
        speeds = [int(x) for x in re.findall('\d+', speed)]
        if speeds:
            return max(speeds)
    return 0


def write_features(all_nodes_file):
    """
    Adds relevant features from open street maps
    """

    all_node_results = fiona.open(all_nodes_file)
    features = []

    # Go through the rest of the nodes, and add any of them that have
    # (hardcoded) open street map features that we care about
    # For the moment, all_nodes only contains street nodes, so we'll
    # only look at signals and crosswalks
    for node in all_node_results:

        if node['properties']['highway'] == 'crossing':
            features.append(geojson.Feature(geometry=geojson.Point(node['geometry']['coordinates']), id=node['properties']['osmid'], properties={'feature': 'crosswalk'}))

        elif node['properties']['highway'] == 'traffic_signals':
            features.append(geojson.Feature(geometry=geojson.Point(node['geometry']['coordinates']), id=node['properties']['osmid'], properties={'feature': 'signal'}))

    features = geojson.FeatureCollection(features)

    with open(os.path.join(MAP_DIR, 'features.geojson'), "w") as f:
        json.dump(features, f)


def get_direction(x, y):
    # Pass in a vector with direction (x, y), return the general direction that vector runs along

    # Numpy convention uses y, x for arctan.
    angle = np.arctan2(y, x)

    # Convert angle to degrees for easier visualisation
    angle = angle * 180 / np.pi

    # 0 is along the positive x-axis, 90 along the positive y-axis, -90 along the negative y-axis etc.
    # Splitting the angles between E-W [east-west], NE-SW (northeast-southwest), N-S (north-south) and NW-SE (northwest-southeast)
    angles = [-157.5, -112.5, -67.5, -22.5, 22.5, 67.5, 112.5, 157.5]

    if angles[0] < angle < angles[1] or angles[4] < angle < angles[5]:
        return 'NE-SW'
    elif angles[1] < angle < angles[2] or angles[5] < angle < angles[6]:
        return 'N-S'
    elif angles[2] < angle < angles[3] or angles[6] < angle < angles[7]:
        return 'NW-SE'
    elif angles[3] < angle < angles[4] or angle > angles[7] or angle < angles[0]:
        return 'E-W'

if not os.path.exists(os.path.join(MAP_DIR, 'osm_ways.shp')):
    print('Generating map from open street map...')
    simple_get_roads(config, True)

# Begin by cleaning up ways data, adding some derived features [e.g. new ID field], finding which roads intersect which nodes
# Then write out osm_elements.geojson as a combination of osm_ways and osm_nodes.
if not os.path.exists(os.path.join(MAP_DIR, 'osm_elements.geojson')):
    print("Cleaning and writing to {}...".format('osm_elements.geojson'))

    clean_and_write(
        os.path.join(MAP_DIR, 'osm_ways.shp'),
        os.path.join(MAP_DIR, 'osm_nodes.shp'),
        os.path.join(MAP_DIR, 'osm_elements.geojson'),
        DOC_DIR, True
    )

# Look through the full all_nodes shp file.
# Add features for any traffic signals or road crossings found at these nodes
# Write out these features to features.geojson
if not os.path.exists(os.path.join(MAP_DIR, 'features.geojson')) or args.forceupdate:
    print('Writing out features.geojson from all_nodes')
    all_nodes_path = os.path.join(MAP_DIR, 'all_nodes', 'nodes', 'nodes.shp')
    write_features(all_nodes_path)<jupyter_output><empty_output><jupyter_text>Create segments<jupyter_code>import rtree
import json
import copy
from shapely.ops import unary_union
from collections import defaultdict
import argparse
import os
import geojson
import re
import sys
from shapely.geometry import MultiLineString, LineString

import util
from segment import Segment


def create_segments_from_json(OSM_elements_path, MAP_DIR):
    # Returns non_int_w_ids and union_inter
    # non_int_w_ids are all the segments that do not have an intersection
    # union_inter are intersection segments where adjacent elements have been combined

    # Return roads [features with type = 'LineString'] and intersections [features with 'intersection' = 1]
    # We use roads_shp_path = osm_elements.geojson normally
    print("Within create_segments.create_segments_from_json")
    roads, inters = util.get_roads_and_inters(OSM_elements_path)
    print("Read in {} road segments".format(len(roads)))

    # Unique ID did not get included in shapefile, need to add it for adjacency
    for i, road in enumerate(roads):
        road.properties['orig_id'] = int(str(99) + str(i))

    # Creates a list that takes the form {ID: 'street names'}
    # If no OSM ID is present, set the key to 0. This situation may occur
    # when we generate alternate maps from city data
    inters_by_id = {
        x['properties']['osmid'] if 'osmid' in x['properties'] else '0':
        x['properties']['streets'] if 'streets' in x['properties'] else None
        for x in inters
    }

    # Buffer each intersection by 20 meters
    int_buffers = get_intersection_buffers(inters, 20)
    print("Found {} intersection buffers".format(len(int_buffers)))

    # Take in the roads and the intersection buffers just found
    # Output the parts of roads that do / dont overlap with the buffers
    non_int_lines, inter_segments = split_segments(roads, int_buffers)

    # For road lines that do not overlap with intersection, rejig some internal variables.
    # In particular add the following properties:
    # 'id' = ('00' + the row of road), 'inter' (= 0 as these are roads), 'displayname'
    # Give them a display_name and append the result to non_int_w_ids
    non_int_w_ids = []
    for i, line in enumerate(non_int_lines):
        value = copy.deepcopy(line)
        value['type'] = 'Feature'
        value['properties']['id'] = '00' + str(i)
        value['properties']['inter'] = 0
        value['properties']['display_name'] = get_non_intersection_name(line, inters_by_id)

        x, y = util.get_center_point(value)
        x, y = util.reproject([[x, y]], inproj='epsg:3857', outproj='epsg:4326')[0]['coordinates']
        value['properties']['center_y'] = y
        value['properties']['center_x'] = x

        non_int_w_ids.append(value)

    print("Extracted {} non-intersection segments".format(len(non_int_w_ids)))

    # Create union_inter, which consists taking the union of road segments that intersect with intersection buffers
    # In particular, 'id', 'data' and 'display_name' properties are added
    # Planarize intersection segments, turning the list of LineStrings into a MultiLineString
    # Previously, the elements of inter_segments['lines'] may have been multiple shapely lines.
    union_inter = []
    print('Now taking the union of segments that overlap an intersection buffer')
    print('Note during this step we set intersection=1 for the segments. Investigate why it was not set earlier')
    for idx, lines in list(inter_segments['lines'].items()):

        lines = unary_union(lines)
        coords = []

        # Fix some mislabelling of dead-end node as intersection due to issue with OSMX
        if type(lines) == LineString:
            lines = MultiLineString([lines.coords])
        for line in lines:
            coords += [[x for x in line.coords]]

        # Assign name to intersection
        name = get_intersection_name(inter_segments['data'][idx])

        # Add the number of segments coming into this intersection to the segment_data
        # Also add the display name of the intersection to each of the segments
        # Also manually set intersection to 1. Should have already been this, but wasn't.
        # Not sure why, look into it later.
        segment_data = []
        for segment in list(inter_segments['data'][idx]):
            segment['intersection_segments'] = len(inter_segments['data'][idx])
            segment['intersection'] = 1
            segment['display_name'] = name
            segment_data.append(segment)

        # Include the id and display name here also. This is historic, and only left to make sure nothing breaks
        # Makes our value have the form {'id': 00, 'data': {... display_name: 'punt rd' ...}, 'display_name': 'punt rd'}
        properties = {
            'id': idx,
            'data': segment_data,
            'display_name': name
        }

        value = geojson.Feature(geometry=geojson.MultiLineString(coords), id=idx, properties=properties)

        x, y = util.get_center_point(value)
        x, y = util.reproject([[x, y]], inproj='epsg:3857', outproj='epsg:4326')[0]['coordinates']
        value['properties']['center_x'] = x
        value['properties']['center_y'] = y

        union_inter.append(value)

    return non_int_w_ids, union_inter


def get_intersection_buffers(intersections, intersection_buffer_units, debug=False, load_buffer_path=False):
    """
    Buffers intersection according to proj units
    Args:
        intersections
        intersection_buffer_units - in meters
        debug - if true, will output the buffers to file for debugging
    Returns:
        a list of polygons, buffering the intersections
        these are circles, or groups of overlapping circles
    """

    print('Within create_segments.get_intersection_buffers.')
    print('Many intersections are not correctly labelled by OSM.')

    if not load_buffer_path:
        # Uses .buffer from shapely to buffer the intersection geometry, creating buffered polygons around all intersections
        buffered_intersections = [intersection['geometry'].buffer(intersection_buffer_units) for intersection in intersections]

        # Uses unary_union from shapely to get the union of the buffered circles
        buffered_intersections = unary_union(buffered_intersections)
        print('Finished buffering intersections.')
    else:
        with open(load_buffer_path, "rb") as fp:
            buffered_intersections = pickle.load(fp)

    # Output the buffered shapes for troubleshooting
    if debug:
        util.output_from_shapes(
            [(x, {}) for x in buffered_intersections],
            os.path.join(MAP_DIR, 'int_buffers.geojson')
        )

    # Index the intersection points for for faster lookup in subsequent sections
    print('Creating a spatial index to allow for faster lookups of intersections')
    inter_index = rtree.index.Index()
    for idx, inter_point in enumerate(intersections):
        inter_index.insert(idx, inter_point['geometry'].bounds)

    # Get the intersections that overlap with other intersection buffers and collating, appending to results
    print('Checking which intersections overlap buffers and creating results list of the form [[buffer, matching_intersections]]')
    results = []
    for buff in buffered_intersections:
        matches = []
        for idx in inter_index.intersection(buff.bounds):
            if intersections[idx]['geometry'].within(buff):
                matches.append(intersections[idx]['geometry'])
        results.append([buff, matches])
    print('Finished associating intersections with buffers.')

    return results


def split_segments(roads, int_buffers):
    """
    Finds which parts of each segment either overlaps with a buffered intersection,
    or doesn't. Returns non_int_lines and inter_segments
    Args:
        roads - a list of tuples of shapely shape and dict of segment info
        int_buffers - a list of polygons that buffer intersections
    Returns:
        tuple consisting of:
            non_int_lines - list in same format as input roads, just a subset
                Each element in the list is a tuple of LineString or MultiLineString and dict of properties

            inter_segments - dict of lists with keys 'data' and 'lines'.
                Each element in the lines list is one of the lines overlapping the intersection buffer
                Each element in the data list is a dict of properties corresponding to the lines
    """

    # Create buffered_lines to hold tuples with buffer and original road
    # Also update the index with the index number and buffer bounds.
    # Road_lines_index will be used to find those roads and their idx that intersect with int_buffers
    road_lines_index = rtree.index.Index()
    buffered_lines = []
    for idx, road in enumerate(roads):
        b = road.geometry.buffer(20)
        buffered_lines.append((b, road))
        road_lines_index.insert(idx, b.bounds)

    # Setting up init parameters to be filled.
    roads_with_int_segments = {}
    inter_segments = {'lines': defaultdict(list), 'data': defaultdict(list)}
    count = 0

    # Iterate through buffered intersections
    # Check if any buffered road intersect with buffered intersection
    # If intersect add road to matched_roads, and add intersecting segment to match_segments
    # Then return roads_with_int_Segments which tracks what intersections roads cross with
    # Util.track prints progress every 1000 steps
    print("Inside create_segments.split_segments. Generating intersection segments")
    for i, int_buffer in enumerate(int_buffers):
        util.track(i, 10000, len(int_buffers))
        match_segments = []
        matched_roads = []
        for idx in road_lines_index.intersection(int_buffer[0].bounds):
            road = roads[idx]
            match_segments.append(Segment(road.geometry.intersection(int_buffer[0]), road.properties))
            matched_roads.append(road)

        # Connect road segments to intersections, combine close intersections
        # Match_segments looks like [segment object 1, segment object 2]
        # int_segments looks like [({segment object 1, segment object 2}, union(segments))]
        # Returns tuple containing set of (set(segments), buffer(unary_unions(segments)))
        int_segments = get_connections(int_buffer[1], match_segments)

        # For each road that intersects with the intersection, create an entry in roads_with_int_segments, attaching int_segments.
        # roads_with_int_segments looks like {'id': [({segment object 1, segment object 2}, unions(segments)), the same again], 'id2: ...}
        # road_with_int_segments is initialised above the loop
        for r in matched_roads:
            if r.properties['id'] not in roads_with_int_segments:
                roads_with_int_segments[r.properties['id']] = []
            roads_with_int_segments[r.properties['id']] += int_segments

        # For each segment in the tuple of int_segments:
        # Add the geometry and properties of the segment to inter_segments['lines' / 'data']
        # inter_segments is initialised above the loop
        for int_segment in int_segments:
            inter_segments['lines'][count] = [x.geometry for x in int_segment[0]]
            inter_segments['data'][count] = [x.properties for x in int_segment[0]]
            count += 1

    # For each road segment, check if it appears within roads_with_int_segments.
    # If it doesn't, then append it to non_int_lines a.k.a non_int_roads as is, no chopping up needed
    # If it does, then chop it up, and append the non-intersecting parts.
    non_int_lines = []
    print("Inside create_segments.split_segments. Generating non-intersection segments")

    for i, road in enumerate(roads):
        util.track(i, 20000, len(roads))

        # No overlap with intersection, append directly
        if road.properties['id'] not in roads_with_int_segments:
            non_int_lines.append(geojson.Feature(geometry=geojson.LineString([x for x in road.geometry.coords]), properties=road.properties))

        # Overlaps with intersection, chop it up and append the non-intersecting area.
        else:
            road_info = roads_with_int_segments[road.properties['id']]
            diff = road.geometry

            # Get the difference between road geometry and the buffered intersection. This will be used to chop out the intersection
            for inter in road_info:
                buffered_int = inter[1]
                diff = diff.difference(buffered_int)

            # Do vector operations to remove the ovelapping section with the buffered intersection
            if 'LineString' == diff.type:
                non_int_lines.append(geojson.Feature(geometry=geojson.LineString([x for x in diff.coords]), properties=road.properties))

            # Same idea for MultiLine string
            elif 'MultiLineString' == diff.type:
                coords = []
                for l in diff:
                    for coord in l.coords:
                        coords.append(coord)
                non_int_lines.append(geojson.Feature(geometry=geojson.LineString(coords), properties=road.properties))

            # No sections of the segment fall outside the intersection, so skip.
            else:
                if len(diff) == 0:
                    continue
                print("{} found, skipping".format(diff.type))

    return non_int_lines, inter_segments


def get_connections(points, segments):
    """
    Gets intersections by looking at the connections between points
    and segments that fall within an intersection buffer
    Args:
        points - a list of points
        segments - a list of segment objects
    Returns:
        A list of tuples for each intersection.
        Each tuple contains a set of segment objects
        and the buffer of the unary_union of the segment objects
        with a little bit of padding, because of a slight precision error
        in shapely operations
    """
    # Create a dict with each intersection point's coords as key
    # The values are the point itself and an empty list that will
    # store all the linestrings with a connection to the point
    inters = []
    for p in points:
        inters.append([p, []])

    # Get a starting list of all lines that touch any of the intersection points
    # i.e. any line in segments that is within 0.0001m of an intersection point will be
    # added to the intersection
    for line in segments:
        for i, (curr_shape, _) in enumerate(inters):
            if line.geometry.distance(curr_shape) < .0001:
                inters[i][1].append(line)
                inters[i][0] = unary_union([inters[i][0], line.geometry])

    # Merge connected components
    # This section looks at all the intersections we have. It checks if any intersections overlap.
    # For intersections which overlap, they, and all their lines, are connected and returned
    resulting_inters = []
    connected_lines = []
    while inters:

        # Gets the first intersection.
        # Connected lines are fist defined as simply the intersection + joined roads
        curr = inters.pop(0)
        connected_lines = set(curr[1])

        # Go through all intersections and check if any intersect with current intersection
        if inters:
            connected = [x[1] for x in inters if x[0].intersects(curr[0])]

            # If any are connected, then add that to the set of connected lines
            if connected:
                connected_lines = set(curr[1] + [x for y in connected for x in y])

        # Redefine intersection list to remove any already merged segments
        inters = [x for x in inters if not x[0].intersects(curr[0])]

        # Append to the result whichever intersection you just merged together
        resulting_inters.append((connected_lines, unary_union([x.geometry for x in connected_lines]).buffer(.001)))

    return resulting_inters


def add_point_based_features(non_inters, inters, jsonfile, feats_filename=None,
                             additional_feats_filename=None, forceupdate=False, verbose=False):
    """
    Add any point-based set of features to existing segment data.
    If it isn't already attached to the segments
    Args:
        non_inters
        inters
        jsonfile - points_joined.json, storing the results of snapping
        feats_filename - geojson file for point-based features data
        additional_feats_filename (optional) - file for additional
            points-based data, in json format
        forceupdate - if True, re-snap points and write to file
    """
    print('Within create_segments.add_point_based_features')
    print('Adding the extra features from total node map, plus any further added point features are included')

    if forceupdate or not os.path.exists(jsonfile):
        features = []
        if feats_filename:
            features = util.read_records_from_geojson(feats_filename)
        if additional_feats_filename:
            features += util.read_records(additional_feats_filename, 'record', verbose)

        print('Snapping {} point-based features'.format(len(features)))
        seg, segments_index = util.index_segments(
            inters + non_inters
        )

        # Add ['properties']['near_id'] to features according to overlapping buffer zones and distances
        util.find_nearest(features, seg, segments_index, 20, type_record=True)

        # Write the new points with IDs to identify their appropriate segments to file.
        print("Output {} point-based features to {}".format(len(features), jsonfile))
        print("This will take some time.")
        with open(jsonfile, 'w') as f:
            json.dump([r.properties for r in features], f)

    else:
        features = util.read_records(jsonfile, verbose, None)
        print("Read {} point-based features from file".format(len(features)))

    # For each feature, find it it is near any segments
    # If it is, make an entry within matches for that segment, and begin incrementing the feat_type
    # That means every time a point is added to a segment, it will increment feat_type
    matches = {}
    for feature in features:
        near = feature.near_id
        feat_type = feature.properties['feature']

        if near:
            if str(near) not in matches:
                matches[str(near)] = {}
            if feat_type not in matches[str(near)]:
                matches[str(near)][feat_type] = 0
            matches[str(near)][feat_type] += 1

    # Having correlated point data to segments, now go through each intersection and add point data
    # Add the point data to all intersections that are combined together
    for i, inter in enumerate(inters):
        if str(inter['properties']['id']) in list(matches.keys()):
            matched_features = matches[str(inter['properties']['id'])]

            for prop in inter['properties']['data']:
                for feat in matched_features:
                    prop[feat] = matched_features[feat]

    # Having correlated point data to segments, now go through each non-intersection and add point data
    for i, non_inter in enumerate(non_inters):
        if str(non_inter['properties']['id']) in list(matches.keys()):
            matched_features = matches[non_inter['properties']['id']]

            n = copy.deepcopy(non_inter)

            for feat in matched_features:
                n['properties'][feat] = matched_features[feat]

            non_inters[i] = n

    return non_inters, inters

def write_segments(non_inters, inters, MAP_FP, PROCESSED_DIR):
    # Writes out non_inters as non_inters_segments.geojson
    # Writes out IDs and properties of intersection as inters_data.json
    # Writes out intersections without peroperties for visualisation as inters_segments.geojson
    # Writes out combined intersection and non-intersection with all properties as inter_and_non_int.geojson

    # Store non-intersection segments
    # Project back into 4326 for storage
    non_inters = util.prepare_geojson(non_inters)
    with open(os.path.join(MAP_FP, 'non_inters_segments.geojson'), 'w') as outfile:
        geojson.dump(non_inters, outfile)

    # Get just the properties for the intersections
    # Store all the intersections in json form like {ID : Properties}
    # Save out to inter_data.json

    # ============== INTER_DATA USED IN AGGREGATION =============
    # inter_data = {str(x['properties']['id']): x['properties']['data'] for x in inters}
    inters_data = util.prepare_geojson(inters)
    with open(os.path.join(MAP_FP, 'inters_segments.geojson'), 'w') as outfile:
        geojson.dump(inters_data, outfile)

    # for x in inter_data_prep:
    #     for y in x['properties']['data']:
    #         y['geometry'] = x['geometry']
    #         y['type'] = x['type']
    #     inter_data[str(x['properties']['id'])] = x['properties']['data']

    # ============== COMBINE INTER AND NON_INTER GEOJSONS ============
    # Store the combined segments with all properties
    # This line was originally here but I believe bug: segments = non_inters['features'] + int_w_ids['features']
    # Instead, change inter_data to geojson then append
    segments = non_inters['features'] + inter_data_prep['features']
    segments_collection = geojson.FeatureCollection(segments)
    with open(os.path.join(MAP_FP, 'inter_and_non_int.geojson'), 'w') as outfile:
        geojson.dump(segments_collection, outfile)

    # ============= SAVE OUT INTERS_SEGMENTS for VIS ===============
    # Store the individual intersections without properties, since QGIS appears
    # to have trouble with dicts of dicts, and viewing maps can be helpful
    int_w_ids = [{
        'geometry': x['geometry'],
        'properties': {
            'id': x['properties']['id'],
            'display_name': x['properties']['display_name'] if 'display_name' in x['properties'] else '',
            'center_x': x['properties']['center_x'] if 'center_x' in x['properties'] else '',
            'center_y': x['properties']['center_y'] if 'center_y' in x['properties'] else ''
        }
    } for x in inters]
    int_w_ids = util.prepare_geojson(int_w_ids)
    
    with open(os.path.join(MAP_FP, 'inters_segments.geojson'), 'w') as outfile:
        geojson.dump(int_w_ids, outfile)



def get_intersection_name(inter_segments):
    """
    Get an intersection name from a set of intersection segment names
    Args:
        inter_segments - a list of properties
    Returns:
        intersection name - a string, e.g. First St and Second St
    """

    streets = []
    # Some open street maps segments have more than one name in them
    for street in [x['name'] if 'name' in x.keys() else None
                   for x in inter_segments]:
        if street:
            if '[' in street:
                streets.extend(re.sub("['\[\]]", '', street).split(', '))
            else:
                streets.append(street)
    streets = sorted(list(set(streets)))

    name = ''
    if not streets:
        return name
    if len(streets) == 2:
        name = streets[0] + " and " + streets[1]
    else:
        name = streets[0] + " near "
        name += ', '.join(streets[1:-1]) + ' and ' + streets[-1]

    return name


def get_non_intersection_name(non_inter_segment, inters_by_id):
    """
    Get non-intersection segment names. Mostly in the form:
    X Street between Y Street and Z Street, but sometimes the
    intersection has streets with two different names, in which case
    it will be X Street between Y Street/Z Street and A Street,
    or it's a dead end, in which case it will be X Street from Y Street
    Args:
        non_inter_segment - a geojson non-intersection segment
        inters_by_id - a dict with OSM node IDs as keys
    Returns:
        The display name string
    """

    properties = non_inter_segment['properties']

    if 'name' not in properties or not properties['name']:
        return ''

    segment_street = properties['name']
    from_streets = None
    to_streets = None
    if properties['from'] in inters_by_id and inters_by_id[properties['from']]:
        from_street = inters_by_id[properties['from']]
        from_streets = from_street.split(', ')

        # Remove any street that's part of the named street sections
        if segment_street in from_streets:
            from_streets.remove(segment_street)
    if properties['to'] in inters_by_id and inters_by_id[properties['to']]:
        to_street = inters_by_id[properties['to']]
        to_streets = to_street.split(', ')

        # Remove any street that's part of the named street sections
        if segment_street in to_streets:
            to_streets.remove(segment_street)

    if not from_streets and not to_streets:
        return segment_street

    from_street = None
    if from_streets:
        from_street = '/'.join(from_streets)
    to_street = None
    if to_streets:
        to_street = '/'.join(to_streets)

    if not to_streets:
        return segment_street + ' from ' + from_street
    if not from_streets:
        return segment_street + ' from ' + to_street

    return segment_street + ' between ' + from_street + \
        ' and ' + to_street

    return segment_street


# Take in osm_elements.geojson and output non-inters and inters
# Non-inters are all the non-intersection segments from the data.
# Inters are all the intersections that have been formed by taking union of overlapping segments near ints.
# In both cases, the segments have had appropriate names attached
# Non_inters has properties: id [= '99' + i], inter [= 0], display_name, center_y, center_x as well as normal way properties
# Inters has properties: id [i], data, display_name, center_x, center_y
elements = os.path.join(MAP_DIR, 'osm_elements.geojson')
non_inters, inters = create_segments_from_json(elements, MAP_DIR)  # Returns non_int_w_ids, union_inter

# Load in the feature file.
# Currently this is populated with extra features from the complete nodes shp file.
# It has populated those nodes that correspond to traffic lights / crosswalks within the complete nodes file.
feats_file = os.path.join(MAP_DIR, 'features.geojson')
additional_feats_file = os.path.join(MAP_DIR, 'additional_features.geojson')

if not os.path.exists(feats_file):
    feats_file = None
if not os.path.exists(additional_feats_file):
    additional_feats_file = None
# Add point based features [e.g. from features.geojson] and associate them with each segment according to proximity
# Returns non_inters and inters, but with the point based features snapped to the appropriate segments
# Writes out points_joined.json
if feats_file or additional_feats_file:
    jsonfile = os.path.join(DATA_DIR, 'processed', 'points_joined.json')
    non_inters, inters = add_point_based_features(non_inters, inters, jsonfile, feats_filename=feats_file, additional_feats_filename=None, forceupdate=True)

# Write out the segments which have been categorised into non-intersection or intersection
# And have also had point data attached to them.
# Writes out non_inters as non_inters_segments.geojson
# Writes out IDs and properties of intersection as inters_data.json
# Writes out intersections without properties for visualisation as inters_segments.geojson
# Writes out combined intersection and non-intersection with all properties as inter_and_non_int.geojson
write_segments(non_inters, inters, MAP_DIR, PROCESSED_DATA_DIR)
# Writes out non_inters as non_inters_segments.geojson
# Writes out IDs and properties of intersection as inters_data.json
# Writes out intersections without peroperties for visualisation as inters_segments.geojson
# Writes out combined intersection and non-intersection with all properties as inter_and_non_int.geojson

# Store non-intersection segments
# Project back into 4326 for storage
non_inters = util.prepare_geojson(non_inters)
with open(os.path.join(MAP_FP, 'non_inters_segments.geojson'), 'w') as outfile:
    geojson.dump(non_inters, outfile)

# Get just the properties for the intersections
# Store all the intersections in json form like {ID : Properties}
# Save out to inter_data.json

# ============== INTER_DATA USED IN AGGREGATION =============
# inter_data = {str(x['properties']['id']): x['properties']['data'] for x in inters}
inters_data = util.prepare_geojson(inters)
with open(os.path.join(MAP_FP, 'inters_segments.geojson'), 'w') as outfile:
    geojson.dump(inters_data, outfile)
# for x in inter_data_prep:
#     for y in x['properties']['data']:
#         y['geometry'] = x['geometry']
#         y['type'] = x['type']
#     inter_data[str(x['properties']['id'])] = x['properties']['data']

# ============== COMBINE INTER AND NON_INTER GEOJSONS ============
# Store the combined segments with all properties
# This line was originally here but I believe bug: segments = non_inters['features'] + int_w_ids['features']
# Instead, change inter_data to geojson then append
segments = non_inters['features'] + inter_data_prep['features']
segments_collection = geojson.FeatureCollection(segments)
with open(os.path.join(MAP_FP, 'inter_and_non_int.geojson'), 'w') as outfile:
    geojson.dump(segments_collection, outfile)

# ============= SAVE OUT INTERS_SEGMENTS for VIS ===============
# Store the individual intersections without properties, since QGIS appears
# to have trouble with dicts of dicts, and viewing maps can be helpful
int_w_ids = [{
    'geometry': x['geometry'],
    'properties': {
        'id': x['properties']['id'],
        'display_name': x['properties']['display_name'] if 'display_name' in x['properties'] else '',
        'center_x': x['properties']['center_x'] if 'center_x' in x['properties'] else '',
        'center_y': x['properties']['center_y'] if 'center_y' in x['properties'] else ''
    }
} for x in inters]
int_w_ids = util.prepare_geojson(int_w_ids)

with open(os.path.join(MAP_FP, 'int_w_ids.geojson'), 'w') as outfile:
    geojson.dump(int_w_ids, outfile)

with open(os.path.join(MAP_FP, 'inters_segments.geojson'), 'r') as fp:
    inters = geojson.load(fp)
inters[0]<jupyter_output><empty_output><jupyter_text>Join crash segments<jupyter_code>from shapely.geometry import Point, shape, mapping, MultiLineString, LineString
import pyproj
def snap_records(combined_seg, segments_index, infile, record_type, verbose=True, startyear=None, endyear=None):

    print("Within join_segments.snap_records")
    print("Reading {} data...".format(record_type))
    records = util.read_records(infile, record_type, verbose, startyear, endyear)

    # Find nearest crashes - 30 tolerance
    # Adds the 'near_id' to our records so that we can correlate with segments
    print("Snapping " + record_type + " records to segments. Tolerance of 30m")
    util.find_nearest(records, combined_seg, segments_index, 30, type_record=True)

    jsonfile = os.path.join(PROCESSED_DIR, record_type + '_joined.json')

    print("output " + record_type + " data to " + jsonfile)
    with open(jsonfile, 'w') as f:
        json.dump([r.properties for r in records], f)
def find_nearest(records, segments, segments_index, tolerance, type_record=False):
    """ Finds nearest segment to records
    tolerance : max units distance from record point to consider

    Adds near_id to records.
    Near_id is the db_segment_id, which comes from segments[segment_id][1]['id']
    """

    print("Using tolerance {}".format(tolerance))

    for record in records:

        # We are in process of transition to using Record class
        # but haven't converted it everywhere, so until we do, need
        # to look at whether the records are of type record or not
        record_point = None
        if type_record:
            record_point = record.point
        else:
            record_point = record['point']

        record_buffer_bounds = record_point.buffer(tolerance).bounds
        nearby_segments = segments_index.intersection(record_buffer_bounds)

        # Populate a new list 'segment_id_with_distance'
        # Include in it all the nearby segments to each record
        # Add the ID and the distance of this segment
        segment_id_with_distance = [
            # Get db index and distance to point
            (
                segments[segment_id][1]['id'],
                segments[segment_id][0].distance(record_point)
            )
            for segment_id in nearby_segments
        ]

        # From 'segment_id_with_distance', find the entry with the smallest distance
        # Populate record['properties']['near_id'] with the segment_id that is closest
        if len(segment_id_with_distance):
            nearest = min(segment_id_with_distance, key=lambda tup: tup[1])
            db_segment_id = nearest[0]
            # Add db_segment_id to record
            if type_record:
                record.near_id = db_segment_id
            else:
                record['properties']['near_id'] = db_segment_id
        # If no segment matched, populate key = ''
        else:
            if type_record:
                record.near_id = ''
            else:
                record['properties']['near_id'] = ''
def read_records(filename, record_type, verbose, startdate=None, enddate=None):
    """
    Reads appropriately formatted json file,
    pulls out currently relevant features,
    converts latitude and longitude to projection 4326, and turns into
    a Crash object
    Args:
        filename - json file
        start - optionally give start for date range of crashes
        end - optionally give end date after which to exclude crashes
    Returns:
        A list of Crashes
    """

    if verbose:
        print('Within util.read_records with the following variables')
        print('Filename:', filename)
        print('Record_type:', record_type)
        print('Startdate:', startdate)
        print('Enddate:', enddate)

    records = []
    items = json.load(open(filename))

    if not items:
        return []

    for item in items:
        record = None
        event = items[item]
        event['ACCIDENT_NO'] = item

        if record_type == 'crash':
            record = Crash(event)
        else:
            record = Record(event)

        records.append(record)

    print("Read in data from {} records".format(len(records)))

    return records
def reproject_records(records, inproj='epsg:4326', outproj='epsg:3857'):
    """
    Reprojects a set of records from one projection to another
    Records can either be points, line strings, or multiline strings
    Args:
        records - list of records to reproject
        inproj - defaults to 4326
        outproj - defaults to 3857
    Returns:
        list of reprojected records
    """
    results = []
    inproj = pyproj.Proj(init=inproj)
    outproj = pyproj.Proj(init=outproj)

    for record in records:

        coords = record['geometry']['coordinates']
        if record['geometry']['type'] == 'Point':
            re_point = pyproj.transform(inproj, outproj, coords[0], coords[1])
            point = Point(re_point)
            results.append({'geometry': point,
                            'properties': record['properties']})
        elif record['geometry']['type'] == 'MultiLineString':
            new_coords = []
            for segment in coords:
                new_segment = []
                for coord in segment:
                    new_segment.append(pyproj.transform(
                        inproj, outproj, coord[0], coord[1]))
                new_coords.append(new_segment)

            results.append({'geometry': MultiLineString(new_coords),
                            'properties': record['properties']})
        elif record['geometry']['type'] == 'LineString':
            new_coords = []
            for coord in coords:
                new_coords.append(
                    pyproj.transform(inproj, outproj, coord[0], coord[1])
                )
            results.append({'geometry': LineString(new_coords),
                            'properties': record['properties']})

    return results
def index_segments(segments, geojson=True, segment=False):
    """
    Reads a list of segments in geojson format, and makes
    a spatial index for lookup
    Args:
        list of segments
        geojson - whether or not the list of tuples are in geojson format
            (the other option is shapely shapes) defaults to True
    Returns:
        segments (in shapely format), and segments_index
    """

    combined_seg = segments
    if segment:
        combined_seg = [(x.geometry, x.properties) for x in segments]
    elif geojson:
        # Read in segments and turn them into shape, propery tuples
        combined_seg = [(shape(x['geometry']), x['properties']) for x in
                        segments]
    # Create spatial index for quick lookup
    segments_index = rtree.index.Index()
    for idx, element in enumerate(combined_seg):
        segments_index.insert(idx, element[0].bounds)

    # print('Within util.index_segments creating segments and segments_index')
    # print('Segments_index is our spatial index:', segments_index)
    # print('Combined_seg is our geojson:', combined_seg)

    return combined_seg, segments_index
    # Reads in inters_segments.geojson and non_inters_segments.geojson
    # Returns all the segments, along with an index that can be used for spatial lookups
    combined_seg, segments_index = util.read_segments(dirname=MAP_DIR)

    # Creates crashes_joined.json, which gives crash data a 'near_id', which maps it to the appropriate OSM segment [not the OSM ID, it uses the ID generated for combined_seg]
    infile = os.path.join(CRASH_DIR, 'crashes.json')
    snap_records(combined_seg, segments_index, infile, record_type='crash', startyear=None, endyear=None)
dirname = MAP_FP
get_inter = True
get_non_inter = True

inter = []
non_inter = []

inter = fiona.open(dirname + '/inters_segments.geojson')
inter_reproj = reproject_records([x for x in inter])

inter_old = inter
inter = inter_reproj

inter = [{
    'geometry': mapping(x['geometry']),
    'properties': x['properties']} for x in inter]

non_inter = fiona.open(dirname + '/non_inters_segments.geojson')
non_inter = reproject_records([x for x in non_inter])
non_inter = [{
    'geometry': mapping(x['geometry']),
    'properties': x['properties']} for x in non_inter]


print("Read in {} intersection, {} non-intersection segments".format(len(inter), len(non_inter)))
segments_list = list(inter) + list(non_inter)

combined_seg, segments_index = index_segments(segments_list)
infile = os.path.join(CRASH_DIR, 'crashes.json')
record_type = 'crash'
verbose=True
startyear=None
endyear=None
print("Within join_segments.snap_records")
print("Reading {} data...".format(record_type))
records = util.read_records(infile, record_type, verbose, startyear, endyear)
records[85].near_id
util.find_nearest(records, combined_seg, segments_index, 30, type_record=True)
records[85].near_id
jsonfile = os.path.join(PROCESSED_DIR, record_type + '_joined.json')

print("output " + record_type + " data to " + jsonfile)
with open(jsonfile, 'w') as f:
    json.dump([r.properties for r in records], f)
records = records
segments = combined_seg
segments_index = segments_index
tolerance=30
type_record=True
# Find nearest
i = 0
for record in records:
    i+=1
    if i == 86:
        
        # We are in process of transition to using Record class
        # but haven't converted it everywhere, so until we do, need
        # to look at whether the records are of type record or not
        record_point = None
        if type_record:
            record_point = record.point
        else:
            record_point = record['point']

        record_buffer_bounds = record_point.buffer(tolerance).bounds
        nearby_segments = segments_index.intersection(record_buffer_bounds)

        # Populate a new list 'segment_id_with_distance'
        # Include in it all the nearby segments to each record
        # Add the ID and the distance of this segment

        segment_id_with_distance = [
            (segments[segment_id][1]['id'], segments[segment_id][0].distance(record_point)) 
            for segment_id in nearby_segments]

        # From 'segment_id_with_distance', find the entry with the smallest distance
        # Populate record['properties']['near_id'] with the segment_id that is closest
        if len(segment_id_with_distance):
            nearest = min(segment_id_with_distance, key=lambda tup: tup[1])
            db_segment_id = nearest[0]
            # Add db_segment_id to record
            if type_record:
                record.near_id = db_segment_id
            else:
                record['properties']['near_id'] = db_segment_id
        # If no segment matched, populate key = ''
        else:
            if type_record:
                record.near_id = ''
            else:
                record['properties']['near_id'] = ''
        print(record)
        print(record.near_id)


# Find nearest crashes - 30 tolerance
# Adds the 'near_id' to our records so that we can correlate with segments
print("Snapping " + record_type + " records to segments. Tolerance of 30m")
util.find_nearest(records, combined_seg, segments_index, 30, type_record=True)

jsonfile = os.path.join(PROCESSED_DIR, record_type + '_joined.json')

print("output " + record_type + " data to " + jsonfile)
with open(jsonfile, 'w') as f:
    json.dump([r.properties for r in records], f)
combined_seg[0]<jupyter_output><empty_output><jupyter_text>Make Canon dataset<jupyter_code>import json
import pandas as pd
import os
import argparse
import numpy as np
import csv
import sys
from datetime import datetime
import pickle
from util import read_geojson
def check_json(json, num=1):
    i = 0
    for j in json:
        if i in range(num):
            print(j)
        i += 1
def get_crash_data(crash_joined_path):

    # Load in crash data
    with open(crash_joined_path, 'r') as f:
        crashes = json.load(f)
    crash = pd.DataFrame(crashes)

    print("Number of read in crashes is:", len(crash))
    # DF cleaning
    # First, drop accidents that don't have a near_id mapping
    # Turn 'near_id' to a string, as it may be mixed datatype here
    # Sort by accident date for data transparency
    # Finally, when crash was turned to JSON the DATE_TIME was turned to UNIX. Turn it back.
    crash['near_id'].replace('', np.nan, inplace=True)
    crash.dropna(inplace=True)
    crash['near_id'] = crash['near_id'].astype(str)
    crash['DATE_TIME'] = pd.to_datetime(crash['DATE_TIME'], unit='ms')
    crash.sort_values('DATE_TIME', inplace=True)

    print("Number of mapped crashes [i.e. given a nearID] after cleaning is:", len(crash))
    return crash


def unnest(df, col, reset_index=False):

    col_flat = pd.DataFrame([[i, x]
                             for i, y in df[col].apply(list).iteritems()
                             for x in y], columns=['I', col])
    col_flat = col_flat.set_index('I')
    df = df.drop(col, 1)
    df = df.merge(col_flat, left_index=True, right_index=True)
    if reset_index:
        df = df.reset_index(drop=True)
    return df


def aggregate(PROCESSED_DIR, keep_feat=None, agg='max'):
    """
    Revised version of aggregate_inter_noninter
    """

    # Read in the combined road feature dataset parameters
    inters_fp = os.path.join(PROCESSED_DIR, 'maps', 'inters_segments.geojson')
    non_inters_fp = os.path.join(PROCESSED_DIR, 'maps', 'non_inters_segments.geojson')

    non_inters = read_geojson(non_inters_fp)
    inters = read_geojson(inters_fp)

    with open(non_inters_fp, 'r') as fp:
        non_inters = geojson.load(fp)

    with open(inters_fp, 'r') as fp:
        inters = geojson.load(fp)

    inters_df = pd.DataFrame(inters['features'], columns=['geometry', 'properties'])
    non_inters_df = pd.DataFrame(non_inters['features'], columns=['geometry', 'properties'])

    properties_flattened = pd.io.json.json_normalize(inters_df['properties'])
    inters_df = pd.concat([inters_df, properties_flattened], axis=1)
    inters_df = unnest(inters_df, 'data')
    inters_df.drop(columns="display_name", inplace=True)
    inters_df.rename(columns={'id': 'inter_id'}, inplace=True)

    data_flattened = pd.io.json.json_normalize(inters_df['data'])
    inters_df.reset_index(drop=True, inplace=True)
    inters_df = pd.concat([inters_df, data_flattened], axis=1)

    geometry_flattened = pd.io.json.json_normalize(inters_df['geometry'])
    geometry_flattened.rename(columns={'type': 'geometry.type'}, inplace=True)
    inters_df = pd.concat([inters_df, geometry_flattened], axis=1)

    inters_df.drop(columns=['properties', 'data', 'geometry'], inplace=True)
    inters_df.set_index('inter_id', inplace=True)

    properties_flattened = pd.io.json.json_normalize(non_inters_df['properties'])
    non_inters_df = pd.concat([non_inters_df, properties_flattened], axis=1)

    geometry_flattened = pd.io.json.json_normalize(non_inters_df['geometry'])
    geometry_flattened.rename(columns={'type': 'geometry.type'}, inplace=True)
    non_inters_df = pd.concat([non_inters_df, geometry_flattened], axis=1)

    non_inters_df.drop(columns=["properties", "inter", "geometry"], inplace=True)
    non_inters_df['intersection_segments'] = 0
    non_inters_df['intersection'] = 0

    combined = pd.concat([inters_df, non_inters_df], sort=True)
    combined['type'] = 'feature'
    combined_geojson = df_to_geojson(combined, properties=['intersection'])
    with open(os.path.join(MAP_FP, 'combined.geojson'), 'w') as outfile:
        geojson.dump(combined_geojson, outfile)

    # Get the feature list as the intersection of the passed in feature list and the combined dataframe column names
    # If keep_feat is not passed in, just take the feature list as the full feature list
    if keep_feat:
        print("Trimming the features from OSM to match our keep_feat list")
        OSM_feat = list(set.intersection(set(combined), set(keep_feat)))
    else:
        print("Keeping all features from OSM")
        OSM_feat = list(set(combined))

    # Since there are multiple segments per intersection, we must merge them somehow
    # We aggregate by taking the maximum of all incoming intersections
    aggregated = getattr(combined[OSM_feat].groupby(combined.index), agg)

    # Return aggregation and adjacency info (orig_id)
    print('About to aggregate inter / non-inter dataframes. This takes quite some time')
    print("About to aggregate data. This takes quite some time. Road features that are included: ", ', '.join(OSM_feat))
    aggregated = aggregated()

    # Many NA values, as the intersection / non-intersection data was dealt with differently
    aggregated = aggregated.fillna(0)

    return aggregated, OSM_feat


def aggregate_inter_noninter(PROCESSED_DIR, keep_feat=None, agg='max'):
    """
    Makes road feature df by combining intersections + non-intersecions

    In particular:
    - selects features using the input 'feats'
    - aggregates intersections by taking the maximum for each of the inputs
    - Returns the concat'd and aggregated data as well as the original IDs of all the road segments

    IMPORTANT: if the aggregation type changes, need to also update how aggregation is calculated in src/data/add_map.py
    """

    print('Within make_canon_dataset.aggregate_inter_noninter')

    # Read in the combined road feature dataset parameters
    inters_fp = os.path.join(PROCESSED_DIR, 'inters_data.json')
    non_inters_fp = os.path.join(PROCESSED_DIR, 'maps', 'non_inters_segments.geojson')

    # Create combined road feature dataset
    # Aggregated contains concat'd intersections / non-intersections, as well as filtering chosen features
    # Aggregates intersection features by taking the maximum of all incoming roads.
    # Read in inters data (json), turn into df with inter index
    df_index = []
    df_records = []
    print("In make_canon_dataset.aggregate_inter_noninter")
    print("Reading in:", inters_fp)

    with open(inters_fp, 'r') as f:
        inters = json.load(f)

        # Append each index to dataframe
        # Each intersection has more than one segment
        # Add each segment's properties to df_records
        for idx, lines in inters.items():
            df_records.extend(lines)
            df_index.extend([idx] * len(lines))

    # Having read in all our data, turn it into a df with one index per intersection
    # Keep in mind that intersections can have multiple segments, meaning that there will be multiple entries
    inters_df = pd.DataFrame(df_records, index=df_index)

    # Read in non_inters data
    print("Reading in:", non_inters_fp)
    non_inters = read_geojson(non_inters_fp)
    non_inters_df = pd.DataFrame([x[1] for x in non_inters])
    non_inters_df.set_index('id', inplace=True)

    # Combining intersections and non-intersections
    combined = pd.concat([inters_df, non_inters_df], sort=True)

    # Get the feature list as the intersection of the passed in feature list and the combined dataframe column names
    # If keep_feat is not passed in, just take the feature list as the full feature list
    if keep_feat:
        print("Trimming the features from OSM to match our keep_feat list")
        OSM_feat = list(set.intersection(set(combined), set(keep_feat)))
    else:
        print("Keeping all features from OSM")
        OSM_feat = list(set(combined))

    # Since there are multiple segments per intersection, we must merge them somehow
    # We aggregate by taking the maximum of all incoming intersections
    aggregated = getattr(combined[OSM_feat].groupby(combined.index), agg)

    # Return aggregation and adjacency info (orig_id)
    print('About to aggregate inter / non-inter dataframes. This takes quite some time')
    print("About to aggregate data. This takes quite some time. Road features that are included: ", ', '.join(OSM_feat))
    aggregated = aggregated()

    # Many NA values, as the intersection / non-intersection data was dealt with differently
    aggregated = aggregated.fillna(0)

    return aggregated, OSM_feat



def make_negative_data(crash, roads):
    # If roads is not false, we are providing a real-world representation of road geometries from which we can sample, rather than
    # having to generate them in a dodgy method.

    # A big problem here is in the real-world, there is 1 data-point for every hour, for every road that doesn't see a crash
    # This is obviously impossible to actually use, as it would be a huge number of roads
    # Instead, we have to try to sample what the actual distribution looks like.
    # In doing so, we can uniformly sample some features, while for other features we have to attempt to get accurate real-world distributions.

    # Uniform distribution of date times that start at first crash and ends at last crash
    # Uniform distribution of segments drawn from the crash data
    # Attach date times to segments
    # Get weather data that is relevant to these times / segments, or at least to the hour in Melbourne on that day

    # Begin by getting all the features we can generate uniformly
    # Date, day of week, hour, month, direction. light_cond [dodgy]
    # This may seem like it is biasing the data, but I assure you it isn't [or at least, is only VERY slightly biasing].
    # Think real hard and you'll work out why.
    max_date = crash['DATE_TIME'].max()
    min_date = crash['DATE_TIME'].min()
    days_difference = (max_date - min_date).days

    num_pos_data = len(crash)
    num_neg_data_day = int(np.ceil(num_pos_data / days_difference))

    dates_in_range = pd.date_range(min_date, max_date).tolist()
    uniform_dates_times = pd.DataFrame({'DATE_TIME': dates_in_range * num_neg_data_day})
    uniform_dates_times['DAY_OF_WEEK'] = uniform_dates_times['DATE_TIME'].dt.weekday
    uniform_dates_times['MONTH'] = uniform_dates_times['DATE_TIME'].dt.month

    # Generate the hour uniformly and append it to uniform_dates_times['HOUR']
    uniform_dates_times['HOUR'] = np.random.choice(range(23), len(uniform_dates_times))

    # Use a true sampling of roads if possible.
    # Otherwise generate the direction uniformly and append it to uniform_dates_times['direction']
    sampled_roads = roads.sample(len(uniform_dates_times), replace=True)

    # Trim down data-frames so that they are both the same length
    len_dates = len(uniform_dates_times)
    len_roads = len(sampled_roads)
    min_len = min(len_dates, len_roads)

    uniform_dates_times.drop(uniform_dates_times.index[min_len:], inplace=True)
    sampled_roads.drop(sampled_roads.index[min_len:], inplace=True)

    # Ensure they are they same length
    if len(uniform_dates_times) != len(sampled_roads):
        print('Within make_canon_dataset, but did not properly trim our negative data. Exiting.')
        sys.exit()
    else:
        print('Have trimmed negative dataset. Now contains {} results compared to the {} results of positive data'.format(len(uniform_dates_times), len(crash)))

    # Now join the dates and segment features
    # Will reset index on both so that they share the same index values to merge on
    # https://stackoverflow.com/questions/29576430/shuffle-dataframe-rows
    sampled_roads = sampled_roads.reset_index(drop=True)
    uniform_dates_times = uniform_dates_times.reset_index(drop=True)
    dates_and_roads = pd.concat([uniform_dates_times, sampled_roads], axis=1).sort_values('DATE_TIME')

    dates_and_roads['ACCIDENT_NO'] = ['Neg' + str(x) for x in range(len(dates_and_roads))]
    dates_and_roads['TARGET'] = 0

    return dates_and_roads


def add_accident_count(df, PROCESSED_DIR, forceupdate=False):
    """
    df = crashes dataframe combined with OSM data
    PROCESSED_DIR = data directory path
    """

    # Add columns regarding historical crash counts to each event in crashes.csv
    # Currently 7 day, 30 day, 365 day, 1825 day and 3650 day time periods
    # This takes a long time to run, so try to load instead from csv file if already exists
    accident_count_fp = os.path.join(PROCESSED_DIR, 'crash/accident_count.pk')

    if forceupdate or not os.path.exists(accident_count_fp):
        print('In make_canon_dataset. About to append accident count by time period to crash data. This takes a long time [2 - 3 hours] and is unoptimised. Fix in future.')
        print('Have no built in graceful exit if you wish to leave this, simple cntrl-c. The make_canon_dataset.py is fast up until here regardless.')
        accident_count = {}
        lenDF = len(df)

        for i in range(lenDF):

            accident_no = df.iloc[i].ACCIDENT_NO
            accident_same_location = df[(df.near_id == df.iloc[i].near_id) & (df.TARGET == 1)]
            accident_same_location_date = accident_same_location.DATE_TIME

            current_accident_date = df.iloc[i].DATE_TIME

            past_7_days = current_accident_date - pd.to_timedelta("7day")
            past_30_days = current_accident_date - pd.to_timedelta("30day")
            past_365_days = current_accident_date - pd.to_timedelta("365day")
            past_1825_days = current_accident_date - pd.to_timedelta("1825day")
            past_3650_days = current_accident_date - pd.to_timedelta("3650day")

            idx_in_the_past = [accident_same_location_date < current_accident_date]
            idx_last_7_days = [accident_same_location_date > past_7_days]
            idx_last_30_days = [accident_same_location_date > past_30_days]
            idx_last_365_days = [accident_same_location_date > past_365_days]
            idx_last_1825_days = [accident_same_location_date > past_1825_days]
            idx_last_3650_days = [accident_same_location_date > past_3650_days]

            idx_7 = [x & y for (x, y) in zip(idx_in_the_past, idx_last_7_days)]
            idx_30 = [x & y for (x, y) in zip(idx_in_the_past, idx_last_30_days)]
            idx_365 = [x & y for (x, y) in zip(idx_in_the_past, idx_last_365_days)]
            idx_1825 = [x & y for (x, y) in zip(idx_in_the_past, idx_last_1825_days)]
            idx_3650 = [x & y for (x, y) in zip(idx_in_the_past, idx_last_3650_days)]

            accident_last_7_days = sum(sum((idx_7)))
            accident_last_30_days = sum(sum((idx_30)))
            accident_last_365_days = sum(sum((idx_365)))
            accident_last_1825_days = sum(sum((idx_1825)))
            accident_last_3650_days = sum(sum((idx_3650)))

            accident_count[accident_no] = [accident_last_7_days, accident_last_30_days, accident_last_365_days, accident_last_1825_days, accident_last_3650_days]

            if i % 1000 == 0:
                print('{}% done.'.format(i / lenDF * 100))

        with open(accident_count_fp, 'wb') as fp:
            pickle.dump(accident_count, fp)

    # If the file already exists then we can simple read it in.
    # Note the list comprehension is due to the strange saving behaviour of previous section, leaves empty rows
    else:
        with open(accident_count_fp, 'rb') as fp:
            print("Reading in the accident counts for all segments")
            accident_count = pickle.load(fp)

    # Now append the new accident counts to the crashes df
    df['LAST_7_DAYS'] = ''
    df['LAST_30_DAYS'] = ''
    df['LAST_365_DAYS'] = ''
    df['LAST_1825_DAYS'] = ''
    df['LAST_3650_DAYS'] = ''
    LAST_N_DAYS_LIST = ['LAST_7_DAYS', 'LAST_30_DAYS', 'LAST_365_DAYS', 'LAST_1825_DAYS', 'LAST_3650_DAYS']
    # df[LAST_N_DAYS_LIST] = accident_count

    df_accident_count_path = os.path.join(PROCESSED_DIR, 'crash/df_accident_count.pk')

    # Must append the accident count to our df.
    # Doing this takes quite some time, so we save it out once complete / load it in if already exists.

    # Creating and saving out to pickle
    if not os.path.exists(df_accident_count_path) or forceupdate:
        print('Appending the historic accident counts to our dataframe. This also takes ~30 minutes.')

        i = 0
        len_accident_count = len(accident_count)

        for key in accident_count.keys():
            for idx, last_n_days in enumerate(LAST_N_DAYS_LIST):
                df[last_n_days].loc[key] = accident_count[key][idx]
            i += 1
            if i % 10000 == 0:
                print('Appended {}% of results'.format(100 * i / len_accident_count))

        with open(df_accident_count_path, 'wb') as fp:
            pickle.dump(df, fp)

    # Loading in from pickle
    else:
        print('Loading in df_with_accident_count from pickle')
        with open(df_accident_count_path, 'rb') as fp:
            df = pickle.load(fp)

    return df

OSM_feat
# Read in the combined road feature dataset parameters
inters_fp = os.path.join(PROCESSED_DIR, 'maps', 'inters_segments.geojson')
non_inters_fp = os.path.join(PROCESSED_DIR, 'maps', 'non_inters_segments.geojson')

non_inters = read_geojson(non_inters_fp)
inters = read_geojson(inters_fp)

with open(non_inters_fp, 'r') as fp:
    non_inters = geojson.load(fp)

with open(inters_fp, 'r') as fp:
    inters = geojson.load(fp)
    
inters_df = pd.DataFrame(inters['features'], columns=['geometry', 'properties'])
non_inters_df = pd.DataFrame(non_inters['features'], columns=['geometry', 'properties'])

properties_flattened = pd.io.json.json_normalize(inters_df['properties'])
inters_df = pd.concat([inters_df, properties_flattened], axis=1)
inters_df = unnest(inters_df, 'data')
inters_df.drop(columns="display_name", inplace=True)
inters_df.rename(columns={'id': 'correlate_id'}, inplace=True)
data_flattened = pd.io.json.json_normalize(inters_df['data'])
inters_df.reset_index(drop=True, inplace=True)
inters_df = pd.concat([inters_df, data_flattened], axis=1)
geometry_flattened = pd.io.json.json_normalize(inters_df['geometry'])
geometry_flattened.rename(columns={'type': 'geometry.type'}, inplace=True)
inters_df = pd.concat([inters_df, geometry_flattened], axis=1)
inters_df.drop(columns=['properties', 'data', 'geometry'], inplace=True)

properties_flattened = pd.io.json.json_normalize(non_inters_df['properties'])
non_inters_df = pd.concat([non_inters_df, properties_flattened], axis=1)
non_inters_df.rename(columns={'id':'correlate_id'}, inplace=True)
geometry_flattened = pd.io.json.json_normalize(non_inters_df['geometry'])
geometry_flattened.rename(columns={'type': 'geometry.type'}, inplace=True)
non_inters_df = pd.concat([non_inters_df, geometry_flattened], axis=1)
non_inters_df.drop(columns=["properties", "inter", "geometry"], inplace=True)
non_inters_df['intersection_segments'] = 0
non_inters_df['intersection'] = 0

combined = pd.concat([inters_df, non_inters_df], sort=True)
combined['type'] = 'feature'
combined.reset_index(inplace=True)
combined.correlate_id = combined.correlate_id.astype(str)
combined_geojson = df_to_geojson(combined, properties=['intersection'])
with open(os.path.join(MAP_FP, 'combined.geojson'), 'w') as outfile:
    geojson.dump(combined_geojson, outfile)

# Get the feature list as the intersection of the passed in feature list and the combined dataframe column names
# If keep_feat is not passed in, just take the feature list as the full feature list
if keep_feat:
    print("Trimming the features from OSM to match our keep_feat list")
    OSM_feat = list(set.intersection(set(combined), set(keep_feat)))
else:
    print("Keeping all features from OSM")
    OSM_feat = list(set(combined))

# Since there are multiple segments per intersection, we must merge them somehow
# We aggregate by taking the maximum of all incoming intersections
aggregated = getattr(combined[OSM_feat].groupby(combined.correlate_id), agg)

# Return aggregation and adjacency info (orig_id)
print('About to aggregate inter / non-inter dataframes. This takes quite some time')
print("About to aggregate data. This takes quite some time. Road features that are included: ", ', '.join(OSM_feat))
aggregated = aggregated()

print('Going to drop any columns that have > 95% as NA. For others fill the NA with "" ')
perc_na = round(aggregated.isna().sum()/len(aggregated),2)
keep = perc_na[perc_na < 0.95].keys()
print('Dropped {}'.format(set(list(aggregated))-set(keep)))
aggregated = aggregated[keep]

# Many NA values, as the intersection / non-intersection data was dealt with differently
aggregated = aggregated.fillna('')
# Get feature details
cat_feat = config['cat_feat']
cont_feat = config['cont_feat']
keep_feat = config['keep_feat']

# Crash data is read in, sorted by date and any crashes that haven't been mapped to a near_id are dropped
crash = get_crash_data(os.path.join(PROCESSED_DIR, 'crash_joined.json'))


# OSM data is concat'd [inters vs non-inters], and then the intersection features are max-aggregated
# Keep_feat controls which OSM features to keep
aggregated, OSM_feat = aggregate(PROCESSED_DIR=PROCESSED_DIR)

aggregated['correlate_id'] = aggregated.index
aggregated.to_csv(os.path.join(PROCESSED_DIR, 'roads.csv.gz'), compression='gzip')
aggregated_geojson = df_to_geojson(aggregated, properties=['display_name'])
with open(os.path.join(MAP_FP, 'aggregated.geojson'), 'w') as outfile:
    geojson.dump(aggregated_geojson, outfile)
    
crash_OSM_merged = pd.merge(crash, aggregated, left_on='near_id', right_on='correlate_id', how='left')
print("Exporting crash dataset to ", PROCESSED_DIR)
crash_OSM_merged.to_csv(os.path.join(PROCESSED_DIR, 'crash.csv.gz'), compression='gzip')
crash_OSM_merged.to_json(os.path.join(PROCESSED_DIR, 'crashes_min.json'), orient="records")
print("Number of unique road segments: {} Number of unique nearIDs in crashes: {}".format(len(aggregated), crash_OSM_merged.nunique().near_id))

# Set target to 1 for positive example
crash_OSM_merged['TARGET'] = 1
aggregated.to_csv(os.path.join(PROCESSED_DIR, 'roads.csv.gz'), compression='gzip')
with open(os.path.join(PROCESSED_DIR, 'roads.pk'), 'wb') as fp:
    pickle.dump(aggregated, fp)
aggregated_open = pd.read_csv(os.path.join(PROCESSED_DIR, 'roads.csv.gz'), dtype={'correlate_id': str})
aggregated_open.correlate_id.nunique()<jupyter_output><empty_output><jupyter_text>Make negative data<jupyter_code>negative_data = make_negative_data(crash=crash_OSM_merged, roads=aggregated)
# Drop any columns within positive data that don't occur in negative data
neg_cols = list(negative_data)
positive_data = crash_OSM_merged[neg_cols]

# Negative and positive data is concatenated
# We make accident_number the index so that we index properly in add_accident_count
# We don't drop the accident_no column however, as we also refer to this in add_accident_count
data_set = pd.concat([positive_data, negative_data]).set_index('ACCIDENT_NO', drop=False)

# Data set has accident counts attached to it
data_set_with_accident_count = add_accident_count(data_set, PROCESSED_DIR)

# Output canonical dataset
print("Exporting canonical dataset to ", PROCESSED_DIR)
data_set_with_accident_count.to_csv(os.path.join(PROCESSED_DIR, 'canon.csv.gz'), compression='gzip')

data_set_with_accident_count.loc[data_set_with_accident_count['correlate_id']=='002']<jupyter_output><empty_output><jupyter_text>Train Model<jupyter_code>sys.path.append(os.path.join(BASE_DIR, 'src', 'models'))
from model_utils import format_crash_data
from model_classes import Indata, Tuner, Tester
import sklearn.linear_model as skl
import numpy as np
import pandas as pd
import scipy.stats as ss
import os
import json
import argparse
import yaml
import sys
DATA_DIR = os.path.join(BASE_DIR, 'data', config['name'])
PROCESSED_DATA_DIR = os.path.join(BASE_DIR, 'data', config['name'], 'processed/')
merged_data_path = os.path.join(PROCESSED_DATA_DIR, config['merged_data'])
def get_features(config, data, datadir):
    """
    Get features from the feature list created during data generation / or the list specified during init_city.
    """

    cont_feat = config['cont_feat']
    cat_feat = config['cat_feat']

    # Dropping continuous features that don't exist
    cont_feat_found = []
    for f in cont_feat:
        if f not in data.columns.values:
            print("Feature " + f + " not found, skipping")
        else:
            cont_feat_found.append(f)

    # Dropping categorical features that don't exist
    cat_feat_found = []
    for f in cat_feat:
        if f not in data.columns.values:
            print("Feature " + f + " not found, skipping")
        else:
            cat_feat_found.append(f)

    # Create featureset holder
    features = cont_feat_found + cat_feat_found

    return cont_feat_found, cat_feat_found, features


def process_features(data, features, config, f_cat, f_cont):

    print('Within train_model.process_features')

    # Features for linear model
    linear_model_features = features

    # Turn categorical variables into one-hot representation through get_dummies
    # Append the newly named one-hot variables [e.g. hwy_type23] to our data frame
    # Append the new feature names to our feature list
    # For linear model features leave out the first one [e.g. hwy_type0 for intercept (?)]
    print('Processing categorical variables [one-hot encoding]')
    for f in f_cat:
        temp = pd.get_dummies(data[f])
        temp.columns = [f + '_' + str(c) for c in temp.columns]
        data = pd.concat([data, temp], axis=1)
        features += temp.columns.tolist()
        linear_model_features += temp.columns.tolist()[1:]

    # Turn continuous variables into their log [add one to avoid -inf errors]
    # Using 1.0 rather than 1 to typecast as float rather than int. This is required for log transfrom.
    # Append new feature name to relevant lists[e.g. log_width]
    print('Processing continuous variables [log-transform]')
    for f in f_cont:
        data['log_%s' % f] = np.log(data[f] + 1.0)
        features += ['log_%s' % f]
        linear_model_features += ['log_%s' % f]

    # Remove duplicated features
    # e.g. if features = ['a', 'b', 'c', 'b'], f_cat = ['a'] and f_cont=['b']
    # then set(features) = {'a', 'b', 'c'} and set(f_cat + f_cont) = {'a', 'b'}

    features = list(set(features) - set(f_cat + f_cont))
    linear_model_features = list(set(linear_model_features) - set(f_cat + f_cont))

    return data, features, linear_model_features


def output_importance(trained_model, features, datadir):
    # output feature importances or coefficients
    if hasattr(trained_model, 'feature_importances_'):
        feature_imp_dict = dict(zip(features, trained_model.feature_importances_.astype(float)))
    elif hasattr(trained_model, 'coefficients'):
        feature_imp_dict = dict(zip(features, trained_model.coefficients.astype(float)))
    else:
        return("No feature importances/coefficients detected")
    # conversion to json
    with open(os.path.join(datadir, 'feature_importances.json'), 'w') as f:
        json.dump(feature_imp_dict, f)


def set_params():

    # cv parameters
    cvp = dict()
    cvp['pmetric'] = 'roc_auc'
    cvp['iter'] = 5  # number of iterations
    cvp['folds'] = 5  # folds for cv (default)
    cvp['shuffle'] = True

    # LR parameters
    mp = dict()
    mp['LogisticRegression'] = dict()
    mp['LogisticRegression']['penalty'] = ['l2']
    mp['LogisticRegression']['C'] = ss.beta(a=5, b=2)  # beta distribution for selecting reg strength
    mp['LogisticRegression']['class_weight'] = ['balanced']
    mp['LogisticRegression']['solver'] = ['lbfgs']

    # xgBoost model parameters
    mp['XGBClassifier'] = dict()
    mp['XGBClassifier']['max_depth'] = list(range(3, 7))
    mp['XGBClassifier']['min_child_weight'] = list(range(1, 5))
    mp['XGBClassifier']['learning_rate'] = ss.beta(a=2, b=15)

    # cut-off for model performance
    # generally, if the model isn't better than chance, it's not worth reporting
    perf_cutoff = 0.5
    return cvp, mp, perf_cutoff


def initialize_and_run(data_model, features, linear_model_features, datadir, target, seed=None):

    print('Within train_model.initialize_and_run')
    print('Will now set initial model parameters, created our InData object, split into train / test sets')

    # Cross-validation parameters, model parameters, perf_cutoff
    cvp, mp, perf_cutoff = set_params()

    # Initialize data with __init__ method
    # Parameters (self, data, target, scoring=None
    # Returns object with properties: .data, .target, .scoring [if provided]
    # With attributes: scoring, data, train_x, train_y, test_x, test_y, is_split
    df = Indata(data_model, target)

    # Create train/test split
    # Parameters (self, pct, datesort=None, group_col=None, seed=None)
    df.tr_te_split(.7, seed=seed)

    # Create weighting variable and attach to parameters
    # This is intended to weight data if it is imbalanced.
    # a[0] = frequency of negative class, a[1] = frequency of positive class
    # normalize = True means .value_counts returns relative frequencies, not absolute count
    a = data_model[target].value_counts(normalize=True)
    w = 1 / a[1]
    mp['XGBClassifier']['scale_pos_weight'] = [w]

    # Initialize tuner
    # Tuner takes the attributes [self, indata, best_models=None, grid_results=None)]
    print('Having done our base initialisation, we attempt to tune model using tuner object')
    tune = Tuner(df)
    try:
        # Base XGBoost model and then base Logistic Regression model
        # Tuning method has the parameters [self, name, m_name, features, cvparams, mparams]
        tune.tune('XG_base', 'XGBClassifier', features, cvp, mp['XGBClassifier'])
        tune.tune('LR_base', 'LogisticRegression', linear_model_features, cvp, mp['LogisticRegression'])

    except ValueError:
        print('CV fails, likely very few of target available, try rerunning at segment-level')
        raise

    # Initialise and run tester object to find best performing model
    print('Tuning finished, running against test data')
    test = Tester(df)
    test.init_tuned(tune)
    test.run_tuned('LR_base', cal=False)
    test.run_tuned('XG_base', cal=False)

    # choose best performing model
    print('Within train_model. Have instantiated tuner object and completed tuning. Will now iterate over test.rundict to check for best performing model. Test.rundict has len:', len(test.rundict), 'and looks like:', test.rundict)
    best_perf = 0
    best_model = None
    for m in test.rundict:
        if test.rundict[m]['roc_auc'] > best_perf:
            best_perf = test.rundict[m]['roc_auc']
            best_model = test.rundict[m]['model']
            best_model_features = test.rundict[m]['features']

    # Check for performance above certain level
    if best_perf <= perf_cutoff:
        print(('Model performs below AUC %s, may not be usable' % perf_cutoff))

    # Train on full data
    print('Best performance was', best_perf, '\n Best model was', best_model, '\nBest model features were', best_model_features)
    trained_model = best_model.fit(data_model[best_model_features], data_model[target])

    # Output feature importance
    output_importance(trained_model, features, datadir)

    # Save out best model to pickle for later use
    with open(os.path.join(datadir, 'model.pk'), 'wb') as fp:
        pickle.dump(trained_model, fp)

with open(CONFIG_FP) as f:
    config = yaml.safe_load(f)
# Read in data
data = pd.read_csv(merged_data_path)
data.sort_values(['DATE_TIME'], inplace=True)

# Get all features that exist within dataset and are being used
f_cont, f_cat, features = get_features(config, data, PROCESSED_DATA_DIR)
print('Our categorical features are:', f_cat)
print('Our continuous features are:', f_cont)
# Remove features that aren't part of f_cat or f_cont or TARGET
data_model = data[f_cat + f_cont + ['TARGET']]
data_model, features, linear_model_features = process_features(data_model, features, config, f_cat, f_cont)
# Print out various statistics to understand model parameters
print("full features:{}".format(features))
print('\n\n Data_model: \n\n', data_model)
print('\n\n features:', features)
print('\n\n lm_features:', linear_model_features)
print('\n\n Process_DATA_DIR:', PROCESSED_DATA_DIR)

# Save out data_model and the features within
data_model_path = os.path.join(PROCESSED_DATA_DIR, 'myDataModel.csv')
if not os.path.exists(data_model_path) or True:
    data_model.to_csv(data_model_path, index=False)

features_path = data_model_path = os.path.join(PROCESSED_DATA_DIR, 'features.pk')
if not os.path.exists(features_path) or True:
    with open(features_path, 'wb') as fp:
        pickle.dump(features, fp)

initialize_and_run(data_model, features, linear_model_features, PROCESSED_DATA_DIR, target='TARGET')<jupyter_output><empty_output><jupyter_text>Predictions<jupyter_code>from model_utils import format_crash_data
from model_classes import Indata, Tuner, Tester
from train_model import process_features, get_features
import sklearn.linear_model as skl


def predict(trained_model, predict_data, features, data_model_features, DATA_DIR):
    """
    Returns
        nothing, writes prediction segments to file
    """

    # Ensure predict_data has the same columns and column ordering as required by trained_model
    predict_data_reduced = predict_data[data_model_features]
    preds = trained_model.predict_proba(predict_data_reduced)[::, 1]
    predict_data['predictions'] = preds

    predict_data.to_csv(os.path.join(DATA_DIR, 'predictions.csv'), index=False)
    predict_data.to_json(os.path.join(DATA_DIR, 'predictions.json'), orient='index')
    with open(os.path.join(DATA_DIR, 'predictions.pk'), 'wb') as fp:
        pickle.dump(predict_data, fp)
    
    print(type(predict_data.coordinates[0]))


def get_accident_count_recent(predict_data, data):
    data['DATE_TIME'] = pd.to_datetime(data['DATE_TIME'])

    current_date = datetime.now()
    past_7_days = current_date - pd.to_timedelta("7day")
    past_30_days = current_date - pd.to_timedelta("30day")
    past_365_days = current_date - pd.to_timedelta("365day")
    past_1825_days = current_date - pd.to_timedelta("1825day")
    past_3650_days = current_date - pd.to_timedelta("3650day")

    recent_crash_7 = data.loc[data['DATE_TIME'] > past_7_days]
    recent_crash_30 = data.loc[data['DATE_TIME'] > past_30_days]
    recent_crash_365 = data.loc[data['DATE_TIME'] > past_365_days]
    recent_crash_1825 = data.loc[data['DATE_TIME'] > past_1825_days]
    recent_crash_3650 = data.loc[data['DATE_TIME'] > past_3650_days]

    column_names = ['LAST_7_DAYS', 'LAST_30_DAYS', 'LAST_365_DAYS', 'LAST_1825_DAYS', 'LAST_3650_DAYS']
    recent_crashes = [recent_crash_7, recent_crash_30, recent_crash_365, recent_crash_1825, recent_crash_3650]

    for col_name in column_names:
        predict_data[col_name] = ""

    i = 0
    print('About to append recent accident counts. This will take some time.')
    for i in range(len(predict_data)):
        current_segment_id = predict_data.loc[i].segment_id

        for j in range(len(recent_crashes)):

            # Find number of crashes at same segment that have occured in appropriate time period
            recent_crash = recent_crashes[j]
            num_crashes = len(recent_crash.loc[recent_crash['segment_id'] == current_segment_id])

            # Assign this number to predict_data
            col_name = column_names[j]
            predict_data.at[i, col_name] = num_crashes

        if i % 5000 == 0:
            print("Got through {}% of results".format(100 * i / len(predict_data)))

    return predict_data


def add_empty_features(predict_data, features):

    # Read in the features from our modelling dataset
    features_path = os.path.join(PROCESSED_DATA_DIR, 'features.pk')
    with open(features_path, 'rb') as fp:
        data_model_features = pickle.load(fp)

    # Get the difference of features between our modelling dataset and predicting dataset
    # Recast as a list to allow for looping over
    feature_difference = list(set(data_model_features) - set(features))

    # Add features in a loop as python doens't like adding all at one time
    for feat in feature_difference:
        predict_data[feat] = 0

    return predict_data, feature_difference, data_model_features

def process_features(data, features, config, f_cat, f_cont):

    print('Within train_model.process_features')

    # Features for linear model
    linear_model_features = features

    # Turn categorical variables into one-hot representation through get_dummies
    # Append the newly named one-hot variables [e.g. hwy_type23] to our data frame
    # Append the new feature names to our feature list
    # For linear model features leave out the first one [e.g. hwy_type0 for intercept (?)]
    print('Processing categorical variables [one-hot encoding]')
    for f in f_cat:
        temp = pd.get_dummies(data[f])
        temp.columns = [f + '_' + str(c) for c in temp.columns]
        data = pd.concat([data, temp], axis=1)
        features += temp.columns.tolist()
        linear_model_features += temp.columns.tolist()[1:]

    # Turn continuous variables into their log [add one to avoid -inf errors]
    # Using 1.0 rather than 1 to typecast as float rather than int. This is required for log transform [base e].
    # Also requires a temp array to hold the values with a recasting, otherwise numpy tries to do things like 5.log, rather than log(5)
    # Append new feature name to relevant lists[e.g. log_width]
    print('Processing continuous variables [log-transform]')
    for f in f_cont:
        temp_array = np.array((data[f] + 1).values).astype(np.float64)
        data['log_%s' % f] = np.log(temp_array)
        features += ['log_%s' % f]
        linear_model_features += ['log_%s' % f]

    # Remove duplicated features
    # e.g. if features = ['a', 'b', 'c', 'b'], f_cat = ['a'] and f_cont=['b']
    # then set(features) = {'a', 'b', 'c'} and set(f_cat + f_cont) = {'a', 'b'}

    features = list(set(features) - set(f_cat + f_cont))
    linear_model_features = list(set(linear_model_features) - set(f_cat + f_cont))

    return data, features, linear_model_features
road_data_path
# Read in road data. We shall generate a prediction for each segment.
#predict_data = pd.read_csv(road_data_path, dtype={'correlate_id': str})
with open(os.path.join(PROCESSED_DIR, 'roads.pk'), 'rb') as fp:
    predict_data = pickle.load(fp)

# Reset the index so that it can be poperly looped over in the attach accident count phase
predict_data.reset_index(inplace=True, drop=True)
    
# Read in crash data. We shall use this to attach historic accident counts to road data.
data = pd.read_csv(crash_data_path, dtype={'near_id': str})
# Check NA within both DF
predict_na = (predict_data.isna().sum())/len(predict_data)
data_na = (data.isna().sum())/len(data)

data_cols = (data_na < 0.95).keys()
predict_cols = (predict_na < 0.95).keys()

print('Removing {} columns from predict data due to NA'.format(set(list(predict_data)) - set(predict_cols)))
print('Removing {} columns from crash data due to NA'.format(set(list(data)) - set(data_cols)))

predict_data = predict_data[predict_cols]
data = data[data_cols]

predict_data.fillna('', inplace=True)
data.fillna('', inplace=True)
# Attach current date / time data
date_time = datetime.now()
hour = date_time.hour
day = date_time.weekday()
month = date_time.month

predict_data['MONTH'] = month
predict_data['DAY_OF_WEEK'] = day
predict_data['HOUR'] = hour

# Attach accident data
predict_path = os.path.join(PROCESSED_DIR, 'predict.csv.gz')
if not os.path.exists(predict_path) or True:
    predict_data = get_accident_count_recent(predict_data, data)
    predict_data.to_csv(predict_path, index=False, compression='gzip')
else:
    predict_data = pd.read_csv(predict_path, dtype={'correlate_id': str})
    
f_cont, f_cat, features = get_features(config, predict_data, PROCESSED_DATA_DIR)
predict_data, features, _ = process_features(predict_data, features, config, f_cat, f_cont)
predict_data, added_features, data_model_features = add_empty_features(predict_data, features)
# Read in best performing model from train_model
with open(os.path.join(PROCESSED_DATA_DIR, 'model.pk'), 'rb') as fp:
    trained_model = pickle.load(fp)

# Get predictions from model and prediction features
predict(trained_model=trained_model, predict_data=predict_data, features=features, data_model_features=data_model_features, DATA_DIR=DATA_DIR)<jupyter_output><class 'list'><jupyter_text>Visualising road geometries<jupyter_code>predictions_geojson = df_to_geojson(predictions, properties=properties)
predictions_fp = os.path.join(PROCESSED_DATA_DIR, 'preds_final.geojson')
predictions_fp_2 = os.path.join(BASE_DIR, 'reports', 'Melbourne', 'preds.txt')
with open(predictions_fp, "w") as fp:
    geojson.dump(predictions_geojson, fp)
with open(predictions_fp_2, "w") as fp:
    geojson.dump(predictions_geojson, fp)
segments_file = os.path.join(MAP_DIR, "inter_and_non_int.geojson")
if not os.path.exists(segments_file):
    sys.exit("segment file not found at {}, exiting".format(segments_file))
else:
    segments = pd.read_json(segments_file)
print('Within make_preds_viz. About to turn dataframe into geojson. This takes a moderate amount of time [2-3 minutes].')
predictions_geojson = df_to_geojson(df=predictions_merged, properties=keep_cols)

# Output as geojson
predictions_collection = geojson.FeatureCollection(predictions_geojson)
predictions_fp = os.path.join(PROCESSED_DATA_DIR, 'preds_final.geojson')
with open(predictions_fp, "w") as fp:
    geojson.dump(predictions_collection, fp)<jupyter_output><empty_output><jupyter_text>Visualisation<jupyter_code>import argparse
import os
import pandas as pd
import geojson
import sys
import yaml


def df_to_geojson(df, properties):
    """
    Turn a dataframe containing point data into a geojson formatted python dictionary

    df : the dataframe to convert to geojson
    properties : a list of columns in the dataframe to turn into geojson feature properties
    lat : the name of the column in the dataframe that contains latitude data
    lon : the name of the column in the dataframe that contains longitude data
    """

    # create a new python dict to contain our geojson data, using geojson format
    geojson = {'type': 'FeatureCollection', 'features': []}

    # Create a secondary properties list which doesn't include type or coordinates
    properties_update = []
    for prop in properties:
        if prop not in ['type', 'coordinates', 'geometry.type']:
            properties_update.append(prop)

    # loop through each row in the dataframe and convert each row to geojson format
    for _, row in df.iterrows():
        # create a feature template to fill in
        feature = {'type': row['type'],
                   'properties': {},
                   'geometry': {'type': row['geometry.type'],
                                'coordinates': row['coordinates']}
                   }

        # For each column, get the value and add it as a new feature property
        for prop in properties_update:
            feature['properties'][prop] = row[prop]

        # add this feature (aka, converted dataframe row) to the list of features inside our dict
        geojson['features'].append(feature)

    return geojson


# Confirm files exist & load data
predictions_file = os.path.join(DATA_DIR, "predictions.json")
if not os.path.exists(predictions_file):
    sys.exit("Predictions file not found at {}, exiting".format(predictions_file))

# Load the predictions
predictions = pd.read_json(predictions_file, orient='index')
print("Loaded predictions. Found {} datapoints".format(len(predictions)))


# Load the road segments file
segments_file = os.path.join(MAP_DIR, "inter_and_non_int.geojson")
if not os.path.exists(segments_file):
    sys.exit("segment file not found at {}, exiting".format(segments_file))
else:
    segments = pd.read_json(segments_file)


# Post process segments. In particular, json_normalize flattens out the nested json string in segments['features']
segments = segments["features"]
segments = pd.io.json.json_normalize(segments, record_prefix=False)
print('Loaded segments, found {} results'.format(len(segments)))

segments.head()

# Post process predictions. In particular, drop center_x and center_y as this data is in segments, and it looks like the predictions data is buggy here
predictions.drop(['coordinates', 'type', 'geometry.type'], axis=1, inplace=True)
segments.id = segments.id.astype(str)
# Merge the two. This gives the necessary geoJSON features to our predictions DF [though they may already be there]
predictions_merged = pd.merge(predictions, segments, left_on='correlate_id', right_on='id', how='inner')
predictions.loc[predictions['correlate_id'] == '000']




# Rename relevant columns
predictions_merged.rename(columns={'geometry.coordinates': 'coordinates', 'properties.center_x': 'center_x', 'properties.center_y': 'center_y'}, inplace=True)

# Get rid of unwanted columns and rename those that have gained prefixes during flattening.
keep_cols = config['cont_feat'] + config['cat_feat'] + ['segment_id', 'coordinates', 'type', 'geometry.type', 'center_x', 'center_y', 'predictions']
predictions_merged = predictions_merged[keep_cols]

# Get rid of some NA values that occur in 'display_name' and 'node_type_int'
previous_len = len(predictions)
predictions['display_name'].fillna("No name found.", inplace=True)
if pd.isnull(predictions).sum().sum() > 0:
    print('Within make_preds_viz. Some NA values still found within dataframe. Exiting')
    sys.exit()
print('Filled {} out of a total of {} results due to NA values'.format(len(predictions) - previous_len, len(predictions)))

# Turn back into geojson to write out
print('Within make_preds_viz. About to turn dataframe into geojson. This takes a moderate amount of time [2-3 minutes].')
predictions_geojson = df_to_geojson(df=predictions_merged, properties=keep_cols)

# Output as geojson
predictions_collection = geojson.FeatureCollection(predictions_geojson)
predictions_fp = os.path.join(PROCESSED_DATA_DIR, 'preds_final.geojson')
with open(predictions_fp, "w") as fp:
    geojson.dump(predictions_collection, fp)

print("Wrote {} assembled predictions to file {}".format(len(predictions_collection['features']), predictions_fp))<jupyter_output><empty_output><jupyter_text>Make GEOJSON of roads for visualisation  Quick Test<jupyter_code>df = predictions_merged
properties = keep_cols

geojson = {'type': 'FeatureCollection', 'features': []}

# Create a secondary properties list which doesn't include type or coordinates
properties_update = []
for prop in properties:
    if prop not in ['type', 'coordinates', 'geometry.type']:
        properties_update.append(prop)

# Loop through each row in the dataframe and convert each row to geojson format
for _, row in df.iterrows():
    # create a feature template to fill in
    feature = {'type': row['type'],
               'properties': {},
               'geometry': {'type': row['geometry.type'],
                            'coordinates': row['coordinates']}
               }

    # For each column, get the value and add it as a new feature property
    for prop in properties_update:
        feature['properties'][prop] = row[prop]

    # Add this feature (aka, converted dataframe row) to the list of features inside our dict
    geojson['features'].append(feature)

predictions_geojson = geojson
import geojson
# Turn back into geojson to write out
print('Within make_preds_viz. About to turn dataframe into geojson. This takes a moderate amount of time [2-3 minutes].')
predictions_geojson = df_to_geojson(df=predictions_merged, properties=keep_cols)

# Output as geojson
predictions_collection = geojson.FeatureCollection(predictions_geojson)
predictions_fp = os.path.join(PROCESSED_DATA_DIR, 'preds_final.geojson')
with open(predictions_fp, "w") as fp:
    geojson.dump(predictions_geojson, fp)
i = 0
for road in predictions_geojson['features']:
    if road['properties']['segment_id'] == '39613129-387153039-2207661111':
        print(road)

os.getcwd()
crashes_df = pd.read_csv("../data/Melbourne/processed/crash.csv.gz")
    
crashes_df.head()
crashes_df.to_json(orient="index")
crashes_df.to_json('../data/Melbourne/processed/crashes_min_records.json', orient="records")
crashes_df.to_json(orient="records")<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1022663491148496, 'fraction_numerical': 0.006522959299767904, 'mean_word_length': 3.609895104895105, 'pattern_counts': {'":': 8, '<': 72, '<?xml version=': 0, '>': 68, 'https://': 4, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '12', 'dir': 'jupyter-structured-clean-dedup', 'id': '123242', 'license': 'MIT', 'n_tokens_mistral': 41158, 'n_tokens_neox': 39289, 'n_words': 11850, 'path': 'src/End-To-End.ipynb', 'repo_name': 'pankace/MelbourneCrashModel', 'size': '131929'}