<jupyter_start><jupyter_text>Data ExplorationWe will do our data explorationg in here, using the processed and saved dictionaries so that we don't have to parse through the data files every time we want to analyze it.<jupyter_code>import pickle
import matplotlib.pyplot as plt
import numpy as np
import pandas as pd
import seaborn as sns
import scipy.interpolate
%matplotlib inline
with open('FinalTimeSeriesDataset.pickle') as file: 
    dataset, stationslist = pickle.load(file)
#remove stations with problematic data
#for station in stationslist:
#    if (station == 'WALL ST') or (station == 'SUTPHIN BLVD') or (station == 'METROPOLITAN AV')\
#        or (station == 'NEVINS ST') or (station == 'KINGSTON-THROOP'):
#        del dataset[station]
#stationslist = list(dataset.keys())[:]
station = 'CLARK ST'
time = dataset[station].times
pickups = dataset[station].pickups
exits = dataset[station].exits
for count in range(5):
    #We need to standardize the two time series so they can be compared and correlated properly
    pickups = (pickups - pickups.mean())/pickups.std()
    exits = (exits - exits.mean())/exits.std()
    #Now we need to remove outliers (greater than 3-sigma) that could be due to bad data
    pickups[pickups.apply(abs) > 3] = 0
    exits[exits.apply(abs) > 3] = 0
#plotting
plt.xticks(rotation=70)
plt.plot(time, pickups, time, exits);
plt.title(station);
plt.xlabel('Time');
plt.ylabel('Counts (Standardized)');
Correlations = {}
for station in dataset:
    pickups = dataset[station].pickups
    exits = dataset[station].exits
    for count in range(5):
        #We need to standardize the two time series so they can be compared and correlated properly
        pickups = (pickups - pickups.mean())/pickups.std()
        exits = (exits - exits.mean())/exits.std()
        #Now we need to remove outliers (greater than 3-sigma) that could be due to bad data
        pickups[pickups.apply(abs) > 3] = 0
        exits[exits.apply(abs) > 3] = 0
    Corr = pd.Series.corr(pickups,exits)
    if Corr > 0:
        Correlations[station] = Corr
#Graphing
df_corr = pd.DataFrame({'Station' : list(Correlations.keys()), 'Correlation' : list(Correlations.values())})
df_corr = barchart.sort_values(['Correlation'], ascending=False)
df_corr = df_corr[df_corr.Correlation>0.5]
plt.figure(figsize=(20,15))
plt.barh(range(len(df_corr)), df_corr.Correlation, align='center')
plt.yticks(range(len(df_corr)), df_corr.Station, fontsize = 40)
plt.ylabel('Station Names', fontsize = 40)
plt.xlabel('Correlation Between Uber Pickups and Station Exits',fontsize = 30)
plt.ylim(-1,len(df_corr))
plt.xticks(fontsize = 30)
plt.xlim(0.0,0.7)
# Loading the Uber pickups data and their closest stations:
with open('UberData/UberStationPickups.pickle') as file: 
    datalist = pickle.load(file)
    pickups = datalist[0]
lat = []
lon = []
corr = []
for key in stationslist:
    if key in Correlations:
        lat.append( pickups[pickups.Closest_Station == key].Lat.mean() )
        lon.append( pickups[pickups.Closest_Station == key].Lon.mean() )
        corr.append(Correlations[key])
y,x,z = np.array(lat),np.array(lon),np.array(corr)
# Set up a regular grid of interpolation points
xi, yi = np.linspace(x.min(), x.max(), 100), np.linspace(y.min(), y.max(), 100)
xi, yi = np.meshgrid(xi, yi)

# Interpolate
rbf = scipy.interpolate.Rbf(x, y, z, function='linear')
zi = rbf(xi, yi)

plt.figure(figsize=(20,20))
plt.imshow(zi, vmin=z.min(), vmax=z.max(), origin='lower',
           extent=[x.min(), x.max(), y.min(), y.max()],cmap=plt.cm.OrRd)
plt.scatter(x, y, c=z,s=100,cmap=plt.cm.OrRd)
plt.xticks(fontsize = 20)
plt.yticks(fontsize = 20)
plt.colorbar()
plt.savefig('barchart.png')<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.14270328486806677, 'fraction_numerical': 0.014001077005923533, 'mean_word_length': 4.415451895043732, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 11, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '265832', 'license': 'MIT', 'n_tokens_mistral': 1392, 'n_tokens_neox': 1293, 'n_words': 343, 'path': 'Code/DataExploration.ipynb', 'repo_name': 'sandymule/Lyber-MTA', 'size': '3714'}