<jupyter_start><jupyter_text>Comparing Training and Test and Parking and Sensor Datasets<jupyter_code>import sys
import pandas as pd
import numpy as np
import datetime as dt
import time
import matplotlib.pyplot as plt
sys.path.append('../')
from common import reorder_street_block, process_sensor_dataframe, get_train, \
                    feat_eng, add_tt_gps, get_parking, get_test, plot_dataset_overlay, \
                    parking_join_addr, tt_join_nh

%matplotlib inline<jupyter_output><empty_output><jupyter_text>Import City data<jupyter_code>train = get_train()
train = feat_eng(train)<jupyter_output>Time it took: 0.15621113777160645
adding additional features to test and/or train ...

loading parking data 1.7M
loading address data 30K
calculating closest joins
merging
complete
fixing neighborhood string formatting
merging in neighborhood information
Time it took: 0.7253148555755615<jupyter_text>Import Scraped City Data<jupyter_code>city_stats = pd.read_csv('../ref_data/nh_city_stats.txt',delimiter='|')
city_stats.head()<jupyter_output><empty_output><jupyter_text>Import Parking data with Addresses<jupyter_code>clean_park = parking_join_addr(True)
clean_park.min_join_dist.value_counts()
clean_park.head(25)
plot_dataset_overlay()<jupyter_output>loading training and test files
Time it took: 0.13714194297790527
adding additional features to test and/or train ...

loading parking data 1.7M
loading address data 30K
calculating closest joins
merging
complete
fixing neighborhood string formatting
merging in neighborhood information
Time it took: 0.47041797637939453
Time it took: 0.10045909881591797
adding additional features to test and/or train ...

loading parking data 1.7M
loading address data 30K
calculating closest joins
merging
complete
fixing neighborhood string formatting
merging in neighborhood information
Time it took: 0.4604032039642334
loading parking and sensor data
loading cached copy
Time it took: 3.303312063217163
loading cached copy
Time it took: 11.266279935836792
plotting<jupyter_text>Prototyping Below (joining and Mapping Code)<jupyter_code>from multiprocessing import cpu_count, Pool

# simple example of parallelizing filling nulls
def parallelize(data, func):
    cores = cpu_count() 
    
    data_split = np.array_split(data, cores)
    pool = Pool(cores)
    data = np.concatenate(pool.map(func, data_split), axis=0)
    pool.close()
    pool.join()
    return data

def closest_point(park_dist):
    output = np.zeros((park_dist.shape[0], 3), dtype=int)
    for i, point in enumerate(park_dist):
        x,y, id_ = point
        dist = np.sqrt(np.power(gpspts.iloc[:,0]-x,2) + np.power(gpspts.iloc[:,1]-y,2))    
        output[i,:] = (id_,np.argmin(dist),np.min(dist))
    return output


def parking_join_addr(force=False):
    save_path = DATA_PATH + 'P_parking_clean.feather'
    if os.path.isfile(save_path) and force==False:
        print('loading cached copy')
        join_parking_df = pd.read_feather(save_path)
        return join_parking_df
    else:
        parking_df = get_parking()
        park_dist = parking_df.groupby(['lat','lon'])[['datetime']].count().reset_index()[['lat','lon']]
        park_dist['id'] =park_dist.index

        gps2addr = pd.read_csv('../ref_data/clean_parking_gps2addr.txt', delimiter='|')
        keep_cols = ['full_addr','jlat','jlon','nhood','road','zipcode']
        gpspts = gps2addr[['lat','lon']]
        lkup = parallelize(park_dist.values, closest_point)

        lkup_df = pd.DataFrame(lkup)
        lkup_df.columns = ['parking_idx','addr_idx','min_join_dist']

        tmp = park_dist.merge(lkup_df, how='left', left_index=True, right_on='parking_idx')
        tmp = tmp.merge(gps2addr[keep_cols], how='left', left_on='addr_idx', right_index=True)
        join_parking_df = parking_df.merge(tmp, how='left', on=['lat','lon'])

        join_parking_df.to_feather(save_path)
        return join_parking_df
print("loading parking data 1.7M")
parking_df = get_parking()
park_dist = parking_df.groupby(['lat','lon'])[['datetime']].count().reset_index()[['lat','lon']]
park_dist['id'] =park_dist.index

print("loading address data 30K")
gps2addr = pd.read_csv('../ref_data/clean_parking_gps2addr.txt', delimiter='|')
keep_cols = ['full_addr','jlat','jlon','nhood','road','zipcode']
gpspts = gps2addr[['lat','lon']]

x,y,id_= park_dist.iloc[0,:]
dist = np.sqrt(np.power(gpspts.iloc[:,0]-x,2) + np.power(gpspts.iloc[:,1]-y,2))
np.log(dist)
dist = np.sqrt(np.power(gpspts.iloc[:,0]-x,2) + np.power(gpspts.iloc[:,1]-y,2)) 
join_parking_df
lkup_df = pd.DataFrame(lkup)
lkup_df.columns = ['parking_idx','addr_idx']
tmp = park_dist.merge(lkup_df, how='left', left_index=True, right_on='parking_idx')
keep_cols = ['full_addr','jlat','jlon','nhood','road','zipcode']
tmp = tmp.merge(gps2addr[keep_cols], how='left', left_on='addr_idx', right_index=True)
tmp = parking_df.merge(tmp, how='left', on=['lat','lon'])
tmp.isna().sum()
gpspts = gps2addr[['lat','lon']]
park_dist['id'] =park_dist.index
park_dist.head()
park_dist.shape, gpspts.shape
np.concatenate()
from multiprocessing import cpu_count, Pool

# simple example of parallelizing filling nulls
def parallelize(data, func):
    cores = cpu_count() 
    
    data_split = np.array_split(data, cores)
    pool = Pool(cores)
    data = np.concatenate(pool.map(func, data_split), axis=0)
    pool.close()
    pool.join()
    return data

def closest_point(park_dist):
    output = np.zeros((park_dist.shape[0], 2), dtype=int)
    for i, point in enumerate(park_dist):
        x,y, id_ = point
        dist = np.sqrt(np.power(gpspts.iloc[:,0]-x,2) + np.power(gpspts.iloc[:,1]-y,2))
        output[i,:] = (id_,np.argmin(dist))
    return output




lkup
122.465370178
gps2addr[(gps2addr['lon'] <= -122.46537) & (gps2addr['lon'] > -122.4654) ].sort_values('lon')<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.136153450933379, 'fraction_numerical': 0.037506422332591195, 'mean_word_length': 5.014418125643666, 'pattern_counts': {'":': 0, '<': 20, '<?xml version=': 0, '>': 20, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '5', 'dir': 'jupyter-structured-clean-dedup', 'id': '10071', 'license': 'MIT', 'n_tokens_mistral': 2235, 'n_tokens_neox': 2075, 'n_words': 418, 'path': 'eda/06_dataset_comparison.ipynb', 'repo_name': 'timdavidlee/sf-parking-ml', 'size': '5839'}