<jupyter_start><jupyter_text>KDD 2020  Where should I park?Using RAPIDS to find parking spots in Seattle. Load the modules<jupyter_code>!nvidia-smi<jupyter_output>Sat Aug 22 02:15:52 2020       
+-----------------------------------------------------------------------------+
| NVIDIA-SMI 450.51.06    Driver Version: 450.51.06    CUDA Version: 11.0     |
|-------------------------------+----------------------+----------------------+
| GPU  Name        Persistence-M| Bus-Id        Disp.A | Volatile Uncorr. ECC |
| Fan  Temp  Perf  Pwr:Usage/Cap|         Memory-Usage | GPU-Util  Compute M. |
|                               |                      |               MIG M. |
|===============================+======================+======================|
|   0  TITAN RTX           Off  | 00000000:01:00.0 Off |                  N/A |
| 41%   33C    P8    24W / 280W |      0MiB / 24217MiB |      0%      Default |
|                               |                      |                  N/A |
+-------------------------------+----------------------+----------------------+
                                                                               
+-------[...]<jupyter_text>Import modules<jupyter_code>import cudf
from collections import OrderedDict
import numpy as np
import datetime as dt
import matplotlib.pyplot as plt

%load_ext autotime
print(cudf.__version__)<jupyter_output>0.15.0a+4954.ga5dda7faf
time: 443 µs<jupyter_text>Download the dataIf necessary, download the data from my website and unpack. Note -- this may take around 10 minutes depending on the speed of your Internet connection.<jupyter_code>import os

directory = os.path.exists('../data')
archive   = os.path.exists('../data/parking_MayJun2019.tar.gz')
file      = os.path.exists('../data/parking_MayJun2019.csv')
if not directory:
    os.mkdir('../data')

if not archive and not file:
    import wget, shutil
    
    def bar_custom(current, total, width=80):
            print('Downloading: %d%% [%d / %d] bytes' % (current / total * 100.0, current, total))
        
    wget.download('http://tomdrabas.com/data/seattle_parking/parking_MayJun2019.tar.gz')
    shutil.move('parking_MayJun2019.tar.gz', '../data/parking_MayJun2019.tar.gz')
    
if not file:
    import tarfile

    tf = tarfile.open('../data/parking_MayJun2019.tar.gz')
    tf.extractall(path='../data/')<jupyter_output>time: 3.63 ms<jupyter_text>Read the data<jupyter_code>!head -n 10 ../data/parking_MayJun2019.csv
dtypes = OrderedDict([
    ('OccupancyDateTime', 'date'),
    ('PaidOccupancy', 'int64'),
    ('BlockfaceName', 'str'),
    ('SideOfStreet', 'str'),
    ('SourceElementKey', 'int64'),
    ('ParkingTimeLimitCategory', 'int64'),
    ('ParkingSpaceCount', 'int64'),
    ('PaidParkingArea', 'str'),
    ('PaidParkingSubArea', 'str'),
    ('PaidParkingRate', 'int8'),
    ('ParkingCategory', 'str'),
    ('Location', 'str'),
    ('dow', 'int8')
])

df = cudf.read_csv(
    '../data/parking_MayJun2019.csv'
    , skiprows=1
    , dtype=list(dtypes.values())
    , names=list(dtypes.keys())
)

df = df.fillna({'PaidOccupancy': 0, 'ParkingSpaceCount': 999, 'PaidParkingSubArea': 'UKN'})
# size of the file
import os
print('Filesize: {0:.2f}GB'.format(os.path.getsize('../data/parking_MayJun2019.csv') / (1024 ** 3)))
df['PaidOccupancy']     = df['PaidOccupancy'].astype('float64')
df['ParkingSpaceCount'] = df['ParkingSpaceCount'].astype('float64')
df.dtypes
print('The dataset has {0:,} records and {1} columns.'.format(*df.shape))
df.head().to_pandas()<jupyter_output><empty_output><jupyter_text>Extract date information<jupyter_code>df['year'] = df['OccupancyDateTime']._column.year
df['month'] = df['OccupancyDateTime']._column.month
df['day'] = df['OccupancyDateTime']._column.day

df['hour'] = df['OccupancyDateTime']._column.hour
df['minute'] = df['OccupancyDateTime']._column.minute

df[['OccupancyDateTime','year','month','day','hour', 'minute']].head().to_pandas()
counts = df.groupby(['year', 'month', 'day']).agg({'OccupancyDateTime': 'count'})
counts
print('Average number of transactions per day: {0:,.0f}'.format(counts['OccupancyDateTime'].mean()))<jupyter_output>Average number of transactions per day: 954,413
time: 814 µs<jupyter_text>All parking locations<jupyter_code>locations = df[['SourceElementKey', 'BlockfaceName', 'SideOfStreet',
       'ParkingTimeLimitCategory', 'ParkingSpaceCount',
       'PaidParkingArea', 'PaidParkingSubArea',
       'ParkingCategory', 'Location']].drop_duplicates()

locations.head().to_pandas()
print('Number of parking locations in Seattle: {0:,}'.format(locations.shape[0]))
def extractLon(location):
    lon = location.str.extract('([0-9\.\-]+) ([0-9\.]+)')[0]
    return lon#.stod()

def extractLat(location):
    lon = location.str.extract('([0-9\.\-]+) ([0-9\.]+)')[1]
    return lon#.str.stod()

    
locations['longitude'] = extractLon(locations['Location']).astype('float')
locations['latitude'] = extractLat(locations['Location']).astype('float')

locations[['Location', 'longitude', 'latitude']].head().to_pandas()<jupyter_output><empty_output><jupyter_text>Average occupancy<jupyter_code>def avgOccupancy(PaidOccupancy, ParkingSpaceCount, AvgOccupancy):
    for i, (paid, available) in enumerate(zip(PaidOccupancy, ParkingSpaceCount)):
        AvgOccupancy[i] = min(1.0, paid / available) # cap it at 100%, sometimes we see more paid occupancy than spaces available
        
df = (
    df[['OccupancyDateTime', 'PaidOccupancy', 'ParkingSpaceCount'
              , 'SourceElementKey', 'BlockfaceName', 'SideOfStreet'
              , 'ParkingTimeLimitCategory', 'ParkingSpaceCount'
              , 'PaidParkingArea', 'PaidParkingSubArea', 'ParkingCategory', 'dow', 'year', 'month'
              , 'day', 'hour', 'minute']]
    .apply_rows(
        avgOccupancy
        , incols=['PaidOccupancy', 'ParkingSpaceCount']
        , outcols={'AvgOccupancy': np.float64}
        , kwargs={}
    )
)
df.head()
def calcMean(AvgOccupancy, ParkingSpaceCount, MeanOccupancy):
    '''
        Calculate mean
    '''
    for i, (avgOccSum, avgCnt) in enumerate(zip(AvgOccupancy, ParkingSpaceCount)):
        MeanOccupancy[i] = float(avgOccSum) / avgCnt

df_agg_dt = (
    df
    .groupby(['SourceElementKey', 'dow','hour'])
    .agg({
          'ParkingSpaceCount': 'count'
        , 'AvgOccupancy': 'sum'
    })
    .reset_index()
)

df_agg_dt = df_agg_dt.apply_rows(
    calcMean
    , incols=['AvgOccupancy', 'ParkingSpaceCount']
    , outcols={'MeanOccupancy':np.float64}
    , kwargs={}
)

df_agg_dt.drop_column('AvgOccupancy')
df_agg_dt.drop_column('ParkingSpaceCount')

df_agg_dt.head().to_pandas()<jupyter_output><empty_output><jupyter_text>Find the best parking<jupyter_code>from geopy.geocoders import Nominatim

geolocator = Nominatim(user_agent="todrabas_test")
location = geolocator.geocode("400 Broad St, Seattle, WA 98109") # SPACE NEEDLE

locations['LON_Ref'] = location.longitude
locations['LAT_Ref'] = location.latitude
from math import sin, cos, sqrt, atan2, pi

def calculateDistance(latitude, longitude, LAT_Ref, LON_Ref, Distance):
    R = 3958.8 # Earth's radius in miles
    
    for i, (lt, ln, lt_r, ln_r) in enumerate(zip(latitude, longitude, LAT_Ref, LON_Ref)):
        lt_rad = lt / 180.0 * pi
        ln_rad = ln / 180.0 * pi
        
        dlon = (ln_r - ln) / 180.0 * pi
        dlat = (lt_r - lt) / 180.0 * pi
        a = (sin(dlat/2.0))**2 + cos(lt_rad) * cos(lt_rad) * (sin(dlon/2.0))**2
        c = 2 * atan2(sqrt(a), sqrt(1-a))
        distance = R * c
        Distance[i] = distance * 5280 # in feet
        
locations = locations.apply_rows(
    calculateDistance
    , incols=['latitude', 'longitude', 'LAT_Ref', 'LON_Ref']
    , outcols={'Distance':np.float64}
    , kwargs={}
)
# get only meters within 1000 ft
closest = locations.query('Distance < 1000')

closest = (
    closest
    .merge(df_agg_dt, how='inner', on=['SourceElementKey'])
    .query('dow == 3 and hour == 13')
    .sort_values(by='MeanOccupancy')
)

closest_host = closest[['BlockfaceName', 'SideOfStreet'
       , 'ParkingTimeLimitCategory', 'ParkingSpaceCount', 'PaidParkingArea'
       , 'PaidParkingSubArea', 'ParkingCategory', 'Location', 'Distance'
       , 'dow', 'hour', 'MeanOccupancy', 'longitude', 'latitude']
].head().to_pandas()
closest_host<jupyter_output><empty_output><jupyter_text>Plot the parking spots on the mapWe're using gmaps python package that can be found here: https://github.com/pbugnion/gmaps. Follow the instructions contained within the README.md about how to install the package so the map shows properly in jupyter lab.<jupyter_code>closest_host[['BlockfaceName', 'Distance', 'MeanOccupancy']].to_dict('records')
info_box_template = """
<dl>
<dt>Name</dt><dd>{BlockfaceName}</dd>
<dt>Distance</dt><dd>{Distance:.0f}</dd>
<dt>Occupancy (AVG)</dt><dd>{MeanOccupancy:.3f}</dd>
</dl>
"""

parking_info = [info_box_template.format(**parking) for parking in closest_host[['BlockfaceName', 'Distance', 'MeanOccupancy']].to_dict('records')]
import gmaps
from ipywidgets.embed import embed_minimal_html

####################################################
##                                                ##
## CHANGE THE API CREDS IN THE GoogleMapsAPI.cred ##
##                                                ##
####################################################
with open('config/GoogleMapsAPI.cred', 'r') as f:
    gmaps_creds = f.read()

gmaps.configure(api_key=gmaps_creds) # Your Google API key, go to https://console.developers.google.com

parking_layer = gmaps.symbol_layer(
    closest_host[['latitude', 'longitude']], fill_color="green", stroke_color="green", scale=3, info_box_content=parking_info
)

destinations_layer = gmaps.symbol_layer(
    [[location.latitude, location.longitude]]
    , info_box_content=['DESTINATION']
    , scale=5
    , fill_color="red"
    , stroke_color="red"
)

parkings = closest_host.to_dict('records')

lines_layer = gmaps.drawing_layer(features=[
    gmaps.Line(
        start= (parking['latitude'], parking['longitude'])
        , end = (location.latitude, location.longitude)
        , stroke_weight=2
        , stroke_color="red"
    )
    for parking in parkings]
)

fig = gmaps.figure(layout={'height': '500px'})
fig.add_layer(parking_layer)
fig.add_layer(destinations_layer)
fig.add_layer(lines_layer)
embed_minimal_html('maps_rendered/map_as_crow_flies.html', views=[fig])<jupyter_output>time: 67.2 ms
{'fraction_non_alphanumeric': 0.18946457752571375, 'fraction_numerical': 0.024319907718927232, 'mean_word_length': 3.399154334038055, 'pattern_counts': {'":': 0, '<': 47, '<?xml version=': 0, '>': 46, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 1, 'word_list_counts': {'cursed_substrings.json': 17, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '9', 'dir': 'jupyter-structured-clean-dedup', 'id': '580386', 'license': 'Apache-2.0', 'n_tokens_mistral': 3767, 'n_tokens_neox': 3476, 'n_words': 654, 'path': 'the_archive/archived_rapids_event_notebooks/KDD_2020/notebooks/parking/codes/1_rapids_seattleParking.ipynb', 'repo_name': 'luskaner/notebooks-contrib', 'size': '10403'}