<commit_before>#
#  _____   _____  _______ __   _      _______  ______ _______ _____         
# |     | |_____] |______ | \  |         |    |_____/ |_____|   |   |       
# |_____| |       |______ |  \_|         |    |    \_ |     | __|__ |_____  
#                                                                           
# _______  _____  __   _ _    _ _______  ______ _______ _____  _____  __   _
# |       |     | | \  |  \  /  |______ |_____/ |______   |   |     | | \  |
# |_____  |_____| |  \_|   \/   |______ |    \_ ______| __|__ |_____| |  \_|
#                                                                           

# http://www.lfd.uci.edu/~gohlke/pythonlibs/#pyshp
import shapefile

# http://www.lfd.uci.edu/~gohlke/pythonlibs/#pyproj
import pyproj

# http://www.lfd.uci.edu/~gohlke/pythonlibs/#requests
import requests

import hashlib, collections, csv, os, sys, zipfile

from json import dumps, loads

# http://www.codeforamerica.org/specifications/trails/spec.html

TRAILS_URL = 'http://library.oregonmetro.gov/rlisdiscovery/trails.zip'
ORCA_URL = 'http://library.oregonmetro.gov/rlisdiscovery/orca.zip'

WGS84 = pyproj.Proj("+init=EPSG:4326") # LatLon with WGS84 datum used for geojson
ORSP = pyproj.Proj("+init=EPSG:2913", preserve_units=True) # datum used by Oregon Metro

if not os.path.exists(os.getcwd()+'/output'):
    """
    Create a directory to hold the output
    """
    os.makedirs(os.getcwd()+'/output')

def get_duplicates(arr):
    """
    helper function to check for duplicate ids
    """
    dup_arr = arr[:]
    for i in set(arr):
        dup_arr.remove(i)       
    return list(set(dup_arr)) 

def download(path, file):
    if not os.path.exists(os.getcwd()+'/src'):
        os.makedirs(os.getcwd()+'/src')

    with open(os.getcwd()+'/src/'+file+'.zip', 'wb') as handle:
        response = requests.get(path, stream=True)

        if not response.ok:
            # Something went wrong
            print "Failed to download "+file
            sys.exit()

        for block in response.iter_content(1024):
            if not block:
                break

            handle.write(block)
    print 'Downloaded '+file
    unzip(file)
    print 'Unzipped '+file

def unzip(file):
    zfile = zipfile.ZipFile(os.getcwd()+'/src/'+file+'.zip')
    for name in zfile.namelist():
        (dirname, filename) = os.path.split(name)
        zfile.extract(name, os.getcwd()+'/src/')
    zfile.close()

def process_trail_segments():
    stewards = {}
    named_trails = {}
    trail_segments = []

    # read the trails shapefile
    reader = shapefile.Reader(os.getcwd()+'/src/trails.shp')
    fields = reader.fields[1:]
    field_names = [field[0] for field in fields]

    #iterate trails
    for sr in reader.shapeRecords():
    
        atr = dict(zip(field_names, sr.record))

        # we're only allowing open trails to pass
        if atr['STATUS'].upper() == 'OPEN':
            props = collections.OrderedDict()

            # Hash the name and take the last six digits of the hex string
            # In this way we can ensure that agencies get the same id each time
            # without having to maintain an agency id in house.<commit_msg>Update RLISTrails2OT.py<commit_after>#
#  _____   _____  _______ __   _      _______  ______ _______ _____         
# |     | |_____] |______ | \  |         |    |_____/ |_____|   |   |       
# |_____| |       |______ |  \_|         |    |    \_ |     | __|__ |_____  
#                                                                           
# _______  _____  __   _ _    _ _______  ______ _______ _____  _____  __   _
# |       |     | | \  |  \  /  |______ |_____/ |______   |   |     | | \  |
# |_____  |_____| |  \_|   \/   |______ |    \_ ______| __|__ |_____| |  \_|
#                                                                           

# http://www.lfd.uci.edu/~gohlke/pythonlibs/#pyshp
import shapefile

# http://www.lfd.uci.edu/~gohlke/pythonlibs/#pyproj
import pyproj

# http://www.lfd.uci.edu/~gohlke/pythonlibs/#requests
import requests

import hashlib, collections, csv, os, sys, zipfile

from json import dumps, loads

# http://www.codeforamerica.org/specifications/trails/spec.html

TRAILS_URL = 'http://library.oregonmetro.gov/rlisdiscovery/trails.zip'
ORCA_URL = 'http://library.oregonmetro.gov/rlisdiscovery/orca.zip'

WGS84 = pyproj.Proj("+init=EPSG:4326") # LatLon with WGS84 datum used for geojson
ORSP = pyproj.Proj("+init=EPSG:2913", preserve_units=True) # datum used by Oregon Metro

if not os.path.exists(os.getcwd()+'/output'):
    """
    Create a directory to hold the output
    """
    os.makedirs(os.getcwd()+'/output')

def get_duplicates(arr):
    """
    helper function to check for duplicate ids
    """
    dup_arr = arr[:]
    for i in set(arr):
        dup_arr.remove(i)       
    return list(set(dup_arr)) 

def download(path, file):
    if not os.path.exists(os.getcwd()+'/src'):
        os.makedirs(os.getcwd()+'/src')

    with open(os.getcwd()+'/src/'+file+'.zip', 'wb') as handle:
        response = requests.get(path, stream=True)

        if not response.ok:
            # Something went wrong
            print "Failed to download "+file
            sys.exit()

        for block in response.iter_content(1024):
            if not block:
                break

            handle.write(block)
    print 'Downloaded '+file
    unzip(file)
    print 'Unzipped '+file

def unzip(file):
    zfile = zipfile.ZipFile(os.getcwd()+'/src/'+file+'.zip')
    for name in zfile.namelist():
        (dirname, filename) = os.path.split(name)
        zfile.extract(name, os.getcwd()+'/src/')
    zfile.close()

def process_trail_segments():
    stewards = {}
    named_trails = {}
    trail_segments = []

    # read the trails shapefile
    reader = shapefile.Reader(os.getcwd()+'/src/trails.shp')
    fields = reader.fields[1:]
    field_names = [field[0] for field in fields]

    #iterate trails
    for sr in reader.shapeRecords():
    
        atr = dict(zip(field_names, sr.record))

        # we're only allowing open existing trails to pass
        if atr['STATUS'].upper() == 'OPEN' and atr['SYSTEMTYPE'].upper() != 'OPEN':
            props = collections.OrderedDict()

            # Hash the name and take the last six digits of the hex string
            # In this way we can ensure that agencies get the same id each time
            # without having to maintain an agency id in house.
{'fraction_non_alphanumeric': 0.13150940463236438, 'fraction_numerical': 0.005751593346805534, 'mean_word_length': 2.0478446234012315, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 8, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 12, 'word_list_counts': {'cursed_substrings.json': 18, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4070267', 'n_tokens_mistral': 2243, 'n_tokens_neox': 2116, 'n_words': 416}