<jupyter_start><jupyter_text>Intro to RAPIDS using the New York City Yellow Taxi Data light on Data Science, heavy on comparisons.This notebook is for the The Toronto Machine Learning Summit, Nov 16 -29, 2020![TMLS](./img/TMLS.png) This notebook includes* cudf - for basic ETL and some __statistical analysis__ * cuml - for __machine learning__* cugraph - for some __graph analysis__* cuxfilter - for __visualization__ ---- Setup<jupyter_code># load the libraries
import modin.pandas as cudf

import numpy as np
import pandas as pd
import math

import os
import gc

from collections import OrderedDict
import argparse
import datetime
import time
try: 
    import tqdm
except ModuleNotFoundError:
    os.system('pip install tqdm')
    import tqdm
# Let's use Unified Memory (aka managed memory) so that we try and avoid OOM errors 
# start by importing the RAPIDS Memory Manager and then reinitializing with managed memory turn on
import rmm

rmm.reinitialize(   
    managed_memory=True,        # Use managed memory, this allows for oversubscription of the GPU
    pool_allocator=False,       # default is False
    devices=0,                  # GPU device IDs to register. By default, registers only GPU 0.
)<jupyter_output><empty_output><jupyter_text>Download the data<jupyter_code>top_dir = "./"
data_dir = "./nyctaxi"
# Download Taxi data

if os.path.exists(data_dir) == False:
    import nyctaxi_data

    print("downloading data")
    nyctaxi_data.download_nyctaxi_data(["2016"], top_dir)<jupyter_output><empty_output><jupyter_text>---- cuDF - Accelerated Data Frame<jupyter_code># get a list of files
data_path = top_dir + "nyctaxi/2016"

files = []

for f in sorted(os.listdir(data_path)):
    if f[0:6] != 'yellow':
        continue
        
    fname = os.path.join(data_path, f)
            
    files.append(fname)

files
!du -sh $data_path<jupyter_output><empty_output><jupyter_text>Loading data performance test<jupyter_code>def read_pandas(f):
    start_t = time.time()
    df = pd.read_csv(f)
    end_t = time.time() - start_t

    return df, end_t
def read_cudf(f):
    start_t = time.time()
    df = cudf.read_csv(f)
    end_t = time.time() - start_t

    return df, end_t
_ = read_pandas(files[0])
# Load data with Pandas

data = []

start_t = time.time()

for f in files:
    print("\treading " + f, end = '')
    df, t = read_pandas(f)
    print(" ... in time of " + str(t) + " seconds")
    data.append(df)
  
taxi_pdf = pd.concat(data)

end_t = time.time()

print(f"loaded {len(taxi_pdf):,} records in {(end_t - start_t):2f}  seconds")

del data
# Load data with RAPIDS cuDF

data = []

start_t = time.time()

for f in files:
    print("\treading " + f, end = '')
    df, t = read_cudf(f)
    print(" ... in time of " + str(t)+ " seconds")
    data.append(df)

taxi_gdf = cudf.concat(data)

end_t = time.time()

print(f"loaded {len(taxi_gdf):,} records in {(end_t - start_t):2f}  seconds")

del data
taxi_gdf.head(5)<jupyter_output><empty_output><jupyter_text>Sort Comparisons - Single Field<jupyter_code>%%time
sp = taxi_pdf.sort_values(by='trip_distance',ascending=False)
sp.head(5)
%%time
sg = taxi_gdf.sort_values(by='trip_distance',ascending=False)
sg.head(5)<jupyter_output><empty_output><jupyter_text>Group By - Single Column<jupyter_code>%%time
gbp = taxi_pdf.groupby('passenger_count').count()
gbp.head(5)
%%time
gbg = taxi_gdf.groupby('passenger_count').count()
gbg.head(5)<jupyter_output><empty_output><jupyter_text>Fun with Data<jupyter_code>%%time
print(f"Max fare was ${taxi_pdf['fare_amount'].max():,}")
%%time
print(f"Max fare was ${taxi_gdf['fare_amount'].max():,}")
# looking at that huge fare
maxf = taxi_gdf['fare_amount'].max()
taxi_gdf.query('fare_amount == @maxf')
print(f"Farthest trip was {taxi_gdf['trip_distance'].max():,} miles")
# How long did it take to drive that distance?
maxd= taxi_gdf['trip_distance'].max()
taxi_gdf.query('trip_distance == @maxd')<jupyter_output><empty_output><jupyter_text>Changing data types<jupyter_code># change some data types
taxi_gdf = taxi_gdf.astype({'tpep_pickup_datetime':'datetime64[ms]', 'tpep_dropoff_datetime':'datetime64[ms]'})<jupyter_output><empty_output><jupyter_text>Filtering data<jupyter_code># filter out records with missing or outlier values
query_frags = ("(fare_amount > 0 and fare_amount < 500) " +
        "and (passenger_count > 0 and passenger_count < 6) " +
        "and (pickup_longitude > -75 and pickup_longitude < -73) " +
        "and (dropoff_longitude > -75 and dropoff_longitude < -73) " +
        "and (pickup_latitude > 40 and pickup_latitude < 42) " +
        "and (dropoff_latitude > 40 and dropoff_latitude < 42)" +
        "and (pickup_latitude != dropoff_latitude) " +
        "and (pickup_longitude != dropoff_longitude)"
    )

taxi_gdf = taxi_gdf.query(query_frags)<jupyter_output><empty_output><jupyter_text>Add some new features<jupyter_code># easier to reference time by YYYY MM DD version a time stamps
taxi_gdf['hour']  = taxi_gdf['tpep_pickup_datetime'].dt.hour
taxi_gdf['year']  = taxi_gdf['tpep_pickup_datetime'].dt.year
taxi_gdf['month'] = taxi_gdf['tpep_pickup_datetime'].dt.month
taxi_gdf['day']   = taxi_gdf['tpep_pickup_datetime'].dt.day
taxi_gdf['diff']  = taxi_gdf['tpep_dropoff_datetime'].astype('int64') - taxi_gdf['tpep_pickup_datetime'].astype('int64')
def day_of_the_week_kernel(argument):
    day = argument['day']
    month = argument['month']
    year = argument['year']
    if month < 3:
        shift = month
    else:
        shift = 0
    Y = year - (month < 3)
    y = Y - 2000
    c = 20
    d = day
    m = month + shift + 1
    return np.int32((d + math.floor(m * 2.6) + y + (y // 4) + (c // 4) - 2 * c) % 7)
        
taxi_gdf['day_of_week'] = taxi_gdf[['day', 'month', 'year']].apply(day_of_the_week_kernel, axis=1)
taxi_gdf.head(5)<jupyter_output><empty_output><jupyter_text>---  Basic Statistical Data Science Look at some feature - by Hour<jupyter_code># 1) Let's look at a plot of fare by hour
%matplotlib inline
taxi_gdf.groupby('hour').fare_amount.mean().sort_index().plot(legend=True)
# 2) Tips by hour
%matplotlib inline
taxi_gdf.groupby('hour').tip_amount.mean().sort_index().plot(legend=True)
# 3) Number of taxi rides by Hour
%matplotlib inline
taxi_gdf['hour'].groupby(taxi_gdf['hour']).count().sort_index().plot(legend=True)
# Look at what days are the busiest
%matplotlib inline
taxi_gdf.groupby('day_of_week').day_of_week.count().sort_index().plot(legend=True)
# What days have the best tips
%matplotlib inline
taxi_gdf.groupby('day_of_week').tip_amount.mean().sort_index().plot(legend=True)<jupyter_output><empty_output><jupyter_text>Dropping Columns<jupyter_code>taxi_gdf = taxi_gdf.drop('store_and_fwd_flag', axis=1)
taxi_gdf.dtypes<jupyter_output><empty_output><jupyter_text>---  cuML - Accelerated Machine Learning  In Corey's talk --- cuGraph - Accelerated Graph AnalyticsWe need vertex IDs to be integer values but what we have are lat-long pairs (float64).  There are two way that we can address the issue. The hard way and an easy way<jupyter_code>import cugraph
taxi_subset = taxi_gdf[['pickup_longitude', 'pickup_latitude','dropoff_longitude', 'dropoff_latitude', 'trip_distance']].reset_index()
taxi_subset['count'] = 1
del taxi_gdf<jupyter_output><empty_output><jupyter_text>Create vertices and edges the hard way<jupyter_code># create node ID from lat-long combinatiuons
nodes = [
      taxi_subset[['pickup_longitude', 'pickup_latitude']].drop_duplicates().rename(columns={'pickup_longitude': 'long', 'pickup_latitude': 'lat'})
    , taxi_subset[['dropoff_longitude', 'dropoff_latitude']].drop_duplicates().rename(columns={'dropoff_longitude': 'long', 'dropoff_latitude': 'lat'})
]
nodes = cudf.concat(nodes).drop_duplicates().reset_index(drop=True).reset_index().rename(columns={'index': 'id'})
nodes.head(5)
print('Total number of geo points in the dataset: {0:,}'.format(len(nodes)))
edges = (
    taxi_subset[['pickup_longitude', 'pickup_latitude','dropoff_longitude', 'dropoff_latitude', 'trip_distance']]
    .drop_duplicates()
    .rename(columns={'pickup_longitude': 'long', 'pickup_latitude': 'lat'})
    .merge(nodes, on=['lat', 'long'])
    .rename(columns={'long': 'pickup_longitude', 'lat': 'pickup_latitude', 'id': 'pickup_id', 'dropoff_longitude': 'long', 'dropoff_latitude': 'lat'})
    .merge(nodes, on=['lat', 'long'])
    .rename(columns={'long': 'dropoff_longitude', 'lat': 'dropoff_latitude', 'id': 'dropoff_id'})
)[['pickup_id', 'dropoff_id', 'trip_distance']]

edges.head(5)
len(edges)
g = cugraph.Graph()
g.from_cudf_edgelist(edges, source='pickup_id', destination='dropoff_id')<jupyter_output><empty_output><jupyter_text>Pagerank<jupyter_code>%%time
page = cugraph.pagerank(g, alpha=.85, max_iter=1000, tol=1.0e-05)
page.sort_values(by='pagerank', ascending=False).head(5).to_pandas()<jupyter_output><empty_output><jupyter_text>Now the easy way<jupyter_code>g2 = cugraph.Graph()
g2.from_cudf_edgelist(taxi_subset, 
                      source=['pickup_longitude', 'pickup_latitude'], 
                      destination=['dropoff_longitude', 'dropoff_latitude'], 
                      edge_attr='count',
                      renumber=True)
page = cugraph.pagerank(g2, alpha=.85, max_iter=1000, tol=1.0e-05)
page.sort_values(by='pagerank', ascending=False).head(5).to_pandas()<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1407725321888412, 'fraction_numerical': 0.012124463519313304, 'mean_word_length': 5.021317829457364, 'pattern_counts': {'":': 0, '<': 73, '<?xml version=': 0, '>': 71, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '16', 'dir': 'jupyter-structured-clean-dedup', 'id': '119549', 'license': 'Apache-2.0', 'n_tokens_mistral': 3545, 'n_tokens_neox': 3416, 'n_words': 719, 'path': 'conference_notebooks/TMLS_2020/notebooks/Taxi/Overview-Taxi.ipynb', 'repo_name': 'Pahandrovich/notebooks-contrib', 'size': '9320'}