<jupyter_start><jupyter_text>Processing mobility data- Input: the raw mobility data from Moribus (Chicago, Miami, and Boston) e.g. 01_raw/boston_stays_from_esteban.pickle- Outputs: boston_stays, chicago_stays, and Miami_stays.Added time of day, day of week, and other information to the data set. An important change: I replaced the census block group indices by the census tract indices. It is because the predictive performance at the CBG levels is not good.<jupyter_code>import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
import geopandas as gpd
from datetime import datetime 
from dateutil import tz
import gzip
import time
import pickle

# read the boston, miami and chicago data
# it takes several minutes to read the files...
with open("../../data/01_raw/boston_stays_from_esteban.pickle", 'rb') as f:
    df_boston = pickle.load(f) # df_raw is Boston.

df_chicago = pd.read_csv('../../data/01_raw/stays2_16980.csv.gz', compression='gzip')

df_miami = pd.read_csv('../../data/01_raw/stays2_33100.csv.gz', compression='gzip')

# turn them to dic
df_dic = {}
df_dic['boston'] = df_boston
df_dic['chicago'] = df_chicago
df_dic['miami'] = df_miami

df_boston
# print the three shapes
print(df_dic['boston'].shape)
print(df_dic['chicago'].shape)
print(df_dic['miami'].shape)

# Check the number of GEOIDs (census block groups)
print(len(np.unique(df_dic['boston'].GEOID)))
print(len(np.unique(df_dic['boston'].GEOID_home)))
# Check the data info
print(df_dic['boston'].shape)
print(df_dic['boston'].columns)<jupyter_output>(12053805, 14)
Index(['user', 'duration', 'ini_dat', 'lon_medoid', 'lat_medoid', 'GEOID',
       'home_lon_med', 'home_lat_med', 'GEOID_home', 'quant', 'fsq_id', 'cat',
       'distPOI', 'disthome'],
      dtype='object')<jupyter_text>Process stay data<jupyter_code># Need to process the ini_dat to get: hour of day, day of week, weekday vs. weekend.
for key_ in df_dic.keys():
    # Wait, the Chicago time is off by one hour...
    print(key_)
    print(np.min(df_dic[key_]['ini_dat']), np.max(df_dic[key_]['ini_dat']))
    print("Starting time of the data set:", datetime.fromtimestamp(int(np.min(df_dic[key_]['ini_dat']))).strftime('%Y-%m-%d %H:%M:%S'))
    print("Ending time of the data set:",datetime.fromtimestamp(int(np.max(df_dic[key_]['ini_dat']))).strftime('%Y-%m-%d %H:%M:%S'))

    # Use a baseline unix time to start 
    baseline_unix_time = 1474862400 # Mon Sep 26 2016 00:00:00 GMT-0400 (Eastern Daylight Time)
    # one unit in unix time is one second. 
    # 60 seconds, 60 min, 24 hours, 7 days. 

    # get time info
    minutes = (df_dic[key_]['ini_dat'] - baseline_unix_time)/60
    hours = (df_dic[key_]['ini_dat'] - baseline_unix_time)/3600
    days = (df_dic[key_]['ini_dat'] - baseline_unix_time)/(3600*24)

    # hour of day, day of week, etc.
    hour_of_day = (hours.astype(int) + 1)%24 # 0~23 hours
    day_of_week = (days.astype(int) + 1)%7 # 0~6 days; 0: Sunday. 6: Saturday
    weekday = day_of_week.isin([1,2,3,4,5]) # true/false
    weekend = day_of_week.isin([6,0]) # true/false

    # old col names
    old_col_names = list(df_dic[key_].columns)

    # augment time info
    df_dic[key_] = pd.concat([df_dic[key_], hour_of_day, day_of_week, weekday, weekend], axis = 1)
    df_dic[key_].columns = old_col_names + ['hour_of_day', 'day_of_week', 'weekday', 'weekend']
    

df_dic['chicago']
for key_ in df_dic.keys():
    # change GEOIDs to string
    df_dic[key_].GEOID_home = df_dic[key_].GEOID_home.astype('str')
    df_dic[key_].GEOID = df_dic[key_].GEOID.astype('str')

    # add the df CTs.
    # GEOID_ct_list = []
    df_dic[key_]['GEOID'] = df_dic[key_].GEOID.map(lambda x: x[:11])
    df_dic[key_]['GEOID_home'] = df_dic[key_].GEOID_home.map(lambda x: x[:11])

# Quite slow & useless block... 
# current number of census tracts. 983, 988, 1002.
# fast.
print("Boston")
print("Number of activity census tracts: ", len(np.unique(df_dic['boston']['GEOID'])))
print("Number of home census tracts: ", len(np.unique(df_dic['boston']['GEOID_home'])))
print("Total number of census tracts: ", len(set(np.unique(df_dic['boston']['GEOID'])).union(set(np.unique(df_dic['boston']['GEOID_home']))))) 

# number of census tracts. 2206, 
# slow...
print("Chicago")
print("Number of activity census tracts: ", len(np.unique(df_dic['chicago']['GEOID'])))
print("Number of home census tracts: ", len(np.unique(df_dic['chicago']['GEOID_home'])))
print("Total number of census tracts: ", len(set(np.unique(df_dic['chicago']['GEOID'])).union(set(np.unique(df_dic['chicago']['GEOID_home']))))) 

# number of census tracts. 983, 988, 
# slow...
print("Miami")
print("Number of activity census tracts: ", len(np.unique(df_dic['miami']['GEOID'])))
print("Number of home census tracts: ", len(np.unique(df_dic['miami']['GEOID_home'])))
print("Total number of census tracts: ", len(set(np.unique(df_dic['miami']['GEOID'])).union(set(np.unique(df_dic['miami']['GEOID_home'])))))

df_dic['boston']
df_dic['chicago']
df_dic['miami']<jupyter_output><empty_output><jupyter_text>Save<jupyter_code>with open("../../data/02_intermediate/boston_stays.pickle", 'wb') as f:
    pickle.dump(df_dic['boston'], f)

with open("../../data/02_intermediate/chicago_stays.pickle", 'wb') as f:
    pickle.dump(df_dic['chicago'], f)

with open("../../data/02_intermediate/miami_stays.pickle", 'wb') as f:
    pickle.dump(df_dic['miami'], f)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1708533431843369, 'fraction_numerical': 0.023088289619504987, 'mean_word_length': 5.188571428571429, 'pattern_counts': {'":': 0, '<': 12, '<?xml version=': 0, '>': 12, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '3', 'dir': 'jupyter-structured-clean-dedup', 'id': '394168', 'license': 'MIT', 'n_tokens_mistral': 2187, 'n_tokens_neox': 2044, 'n_words': 449, 'path': 'src/02_intermediate/inter_01_process_mobility_data.ipynb', 'repo_name': 'cjsyzwsh/economic_growth_usa', 'size': '5414'}