<jupyter_start><jupyter_text>Libraries<jupyter_code>import numpy as np
from numpy.random import randn

import pandas as pd

#time
from datetime import datetime
from datetime import timedelta

#counting
from collections import Counter
# good old matplotlib
import matplotlib as mpl
import matplotlib.pyplot as plt

#high-level based on matplotlib
import seaborn as sns

#dynamic and interactive
import highcharts
from highcharts.charts import chart<jupyter_output><empty_output><jupyter_text>Init<jupyter_code>sns.set_palette("deep", desat=.6)
sns.set_context(rc={"figure.figsize": (8, 4)})
np.random.seed(1234)
#loading highcharts javascript
highcharts.init()<jupyter_output><empty_output><jupyter_text>Reading data  Checkins<jupyter_code>dfc = pd.read_csv('../../datasets/loc-gowalla_totalCheckins.txt', sep='\t', header=False)
dfc.columns = ['uid','utc','lat','lon','vid']

dfc['utc'] = dfc['utc'].astype('datetime64[ms]')

dfc.head()<jupyter_output><empty_output><jupyter_text>Data munging: cleanup, time parsing, etc. New York, New York! - bounding box: (40.4774, -74.2589), (40.9176, -73.7004)<jupyter_code>ny = (dfc['lat']>=40.4774) & (dfc['lat']<=40.9176) & (dfc['lon']>=-74.2589) & (dfc['lon']<=-73.7004)
dfc = dfc[ny]<jupyter_output><empty_output><jupyter_text>Time columns<jupyter_code>dfc['year']     = dfc['utc'].apply(lambda x: x.date().year)
dfc['month']    = dfc['utc'].apply(lambda x: x.date().month)
dfc['day']      = dfc['utc'].apply(lambda x: x.date().day)

dfc['date']     = dfc['utc'].apply(lambda x: x.date())

def time_in_seconds(x):
    return ((x.hour)*60+x.minute)*60+x.second

dfc['time']     = dfc['utc'].apply(lambda x: time_in_seconds(x.time()))

dfc['isotime']  = dfc['utc'].apply(lambda x: x.isoformat() +'Z')

dfc = dfc[dfc['date']<=datetime(2010,10,19).date()]
dfc[['uid','utc','lat','lon','vid', 'time', 'isotime']].head()<jupyter_output><empty_output><jupyter_text>Venues<jupyter_code>dfv = pd.read_csv('../../datasets/spots.txt', sep='\t', header=False)
dfv.columns = ['vid','name','loc']

coords = dfv['loc'].replace('[^0-9. -]+', '',regex=True)
coords = coords.apply(lambda x: x.split())

dfv['v_lat'] = coords.apply(lambda x: float(x[1]))
dfv['v_lon'] = coords.apply(lambda x: float(x[0]))

dfv = dfv.drop('loc', 1)

ny = (dfv['v_lat']>=40.4774) & (dfv['v_lat']<=40.9176) & (dfv['v_lon']>=-74.2589) & (dfv['v_lon']<=-73.7004)
dfv = dfv[ny]

dfv.head()<jupyter_output><empty_output><jupyter_text>Merge venue names and checkins<jupyter_code>df = pd.merge(dfc, dfv[['vid', 'name']], how='left', on='vid')<jupyter_output><empty_output><jupyter_text>Missing venue names on the available checkins:<jupyter_code>print "missing venue names: {}% of available checkins".format(len(df[pd.isnull(df['name'])])*100 / len(df))<jupyter_output>missing venue names: 19% of available checkins<jupyter_text>Data Exploration<jupyter_code>d = df.groupby('date').size()
chart({
            'chart': {
                'type': 'line',
                'marginRight': 30,
                'marginBottom': 50
            },
            'title': {
                'text':'#checkins per day in New York City'
            },
            'yAxis': {
                'type': 'linear',
                'title': {'text':'#checkins'},
                
            },
            'xAxis': {
                'categories': [str(x) for x in d.index.tolist()]
            },
            'series': [{
                'name': 'date',
                'data': d.tolist()
            }]
})<jupyter_output><empty_output><jupyter_text>Clustering<jupyter_code>from sklearn.cluster import KMeans
start = datetime(2010,1,1,0,0) 
dfw = df[df['utc']>start]
cl = min(200, len(dfw)/8)
ml = KMeans(n_clusters=cl)
ml.fit(dfw[['lat', 'lon']])
labels = Counter(ml.labels_)
clusters = [ [ ml.cluster_centers_[x][1], ml.cluster_centers_[x][0], labels[x] ] for x in labels]
clusters
delta=0.15
chart({
            'chart': {
                'type': 'bubble',
                'zoomType': 'xy',
                'width':800,
                'height':800,
                'plotBackgroundImage':'https://maps.googleapis.com/maps/api/staticmap?center=40.8,-74.0&zoom=11&size=800x800&maptype=roadmap'
            },
            'plotOptions': {
              'bubble': {
                'maxSize':'5%',
                'minSize':'1%'
               }
             },
            'legend': { 
              'enabled': False
            },
            'yAxis': {
              'min':40.8-delta,
              'max':40.8+delta
            },
            'xAxis': {
              'min':-74.0-delta,
              'max':-74.0+delta
            },
            'series': [{'data': clusters, 'color':"#FF0000"}]
}, '800px', '800px')<jupyter_output><empty_output><jupyter_text>Write data to csv file checkins<jupyter_code>cols = ['year','month','day','time', 'isotime','uid','lat','lon','vid']
dfc[cols].to_csv('../../datasets/checkins.csv', header=False, index=False)<jupyter_output><empty_output><jupyter_text>venues<jupyter_code>dfv[['vid','name','v_lat','v_lon']].to_csv('../../datasets/venues.csv', header=False, index=False)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1968365553602812, 'fraction_numerical': 0.032806092560046865, 'mean_word_length': 3.4003436426116838, 'pattern_counts': {'":': 1, '<': 53, '<?xml version=': 0, '>': 53, 'https://': 1, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '12', 'dir': 'jupyter-structured-clean-dedup', 'id': '303761', 'license': 'Unlicense', 'n_tokens_mistral': 2017, 'n_tokens_neox': 1883, 'n_words': 289, 'path': 'notebooks/jupiter/lbsn.ipynb', 'repo_name': 'codeaudit/gowalla-spark-demo', 'size': '5121'}