<jupyter_start><jupyter_text>Scrap live station status data in https://bikeshare.metro.net/stations/json/<jupyter_code>def get_live_station(url):
    
    response = requests.get(url, headers={'User-Agent': 'Mozilla/5.0'})
    stations = response.json()
    
    output = []
    
    for station in stations['features']:
        dict_keys = ['kioskId', 'bikesAvailable', 'docksAvailable', 'name', 'latitude', 'longitude']
        data = {k : station['properties'][k] for k in dict_keys}
        data['time'] = pd.to_datetime('today')
        output.append(data)

    return pd.DataFrame(output)
live_station_df = get_live_station("https://bikeshare.metro.net/stations/json/")
live_station_df.head()<jupyter_output><empty_output><jupyter_text>Remove nano-seconds in time feature<jupyter_code>live_station_df['time'] = live_station_df['time'].astype('datetime64[s]')
# change name of time feature to ds and set it to index
# replace minutes and seconds to 0 
live_station_df.rename(columns={'time':'ds'}, inplace=True)
live_station_df.set_index('ds', inplace=True)
live_station_df.index = live_station_df.index.map(lambda x: x.replace(second=0))
live_station_df.index = live_station_df.index.map(lambda x: x.replace(minute=0))
live_station_df.kioskId = live_station_df.kioskId.astype(str)
live_station_df.head()<jupyter_output><empty_output><jupyter_text>Create new column that concatenated with ds and kioskid to merge it with station_pred_df1<jupyter_code>live_station_df.reset_index(inplace=True)
live_station_df['id'] = live_station_df.ds.astype(str) + '-' + live_station_df.kioskId
live_station_df.head()<jupyter_output><empty_output><jupyter_text>Import forecasted station_pred_df1 that stored as a pickle<jupyter_code>with open('station_pred_df1.pickle', 'rb') as f:
    station_pred_df1 = pickle.load(f)

station_pred_df1.head()
station_pred_df1.tail()
# Take first 4 character in station feature
station_pred_df1.station = station_pred_df1.station.str[:4]<jupyter_output><empty_output><jupyter_text>Create unique id feature by concatenating ds and station_id<jupyter_code>station_pred_df1['id'] = station_pred_df1.ds.astype(str) + '-' + station_pred_df1.station
station_pred_df1.tail()
station_pred_df1.set_index('ds', inplace=True)
merge_live_pred_df = station_pred_df1.merge(live_station_df, on='id')
merge_live_pred_df.head()
merge_live_pred_df.head()
def normalize_demand_number(record):
    if record['y_hat'] >= 4.4:
        return round(record['y_hat']) * 3
    elif record['y_hat'] >= 4:
        return round(record['y_hat']) * 2
    elif record['y_hat'] >= 3:
        return round(record['y_hat']) * 2
    elif record['y_hat'] >= 1:
        return round(record['y_hat']) * 3
    elif record['y_hat'] >= 0.7:
        return round(record['y_hat']) * 2
    elif record['y_hat'] >= 0.4:
        return 1
    else:
        return 0
merge_live_pred_df['demand'] = merge_live_pred_df.apply(normalize_demand_number, axis=1)
merge_live_pred_df
with open('end_station_pred_df.pickle', 'rb') as f:
    end_station_pred_df = pickle.load(f)

end_station_pred_df.rename(columns={'y_hat': 'ys'}, inplace=True)
end_station_pred_df.head()<jupyter_output><empty_output><jupyter_text>Execute the same process for end_station_pred_df<jupyter_code>end_station_pred_df.station = end_station_pred_df.station.str[:4]
end_station_pred_df['id'] = end_station_pred_df.ds.astype(str) + '-' + end_station_pred_df.station
end_station_pred_df.set_index('ds', inplace=True)

merge_live_pred_df = merge_live_pred_df.merge(end_station_pred_df[['id', 'ys']], on='id')
def normalize_ys_number(record):
    if record['ys'] >= 3:
        return round(record['ys']) * 2
    elif record['ys'] >= 1:
        return round(record['ys']) * 3
    elif record['ys'] >= .8:
        return round(record['ys']) * 2
    elif record['ys'] >= .5:
        return round(record['ys']) * 1
    else:
        return 0
merge_live_pred_df['surplus'] = merge_live_pred_df.apply(normalize_ys_number, axis=1)
merge_live_pred_df
def nexthour(record):
    if record['bikesAvailable'] - record['demand'] + record['surplus'] >= 0:
        return record['bikesAvailable'] - record['demand'] + record['surplus']
    else:
        return 0
merge_live_pred_df['nextHour'] = merge_live_pred_df.apply(nexthour, axis=1)
merge_live_pred_df
# df = merge_live_pred_df.copy()
# df.to_csv('df_9_30_5pm.csv')
!ls<jupyter_output>[34mData[m[m                             end_station_pred_df.pickle
EDA_FoliumMap.ipynb              functions.py
LICENSE                          heatmaptime_animation.html
Live_station_status.ipynb        heatmaptimes.html
README.md                        [34mhistory[m[m
TimeSeries_BikeStation.ipynb     markers.html
capstone_EDA.ipynb               station_name.csv
capstone_TimeSeries-Copy1.ipynb  station_name.json
capstone_TimeSeries.ipynb        station_pred_df1.pickle
capstone_TimeSeries_hourly.ipynb<jupyter_text>Create Folium Map that will display available bikes,  open docks, and forecasted available bikes in next hour for each bike stations. Also add minimap using folium plugins<jupyter_code>def get_base_map(df):
    minimap = MiniMap(toggle_display=True, width=300, height=150, zoom_animation=True, zoom_level_offset=-6, position='topleft')
   
    return folium.Map(location=[df.latitude.mean(),
                             df.longitude.mean()],
                   zoom_start=20,
                     tiles='OpenStreetMap').add_child(minimap)
base = get_base_map(merge_live_pred_df)
base
def add_station_markers(initial_map, df):
    #station location visualization
    
    out_map = initial_map
    for lat, lon, Name, Available, Opendocks, Next_hr_remaining, in zip(df['latitude'], df['longitude'], df['name'], 
                                df['bikesAvailable'], df['docksAvailable'], df['nextHour']):
        folium.Marker([lat,lon], popup=(str(Name).capitalize() + '<br>'
                                        '<br><b>Available: </b>' + str(Available) + '<br>'
                                        '<br><b>Opendocks: </b>' + str(Opendocks) + '<br>'
                                        '<br><b>Remaining Next Hour:</b> ' + str(Next_hr_remaining)),
                     icon=folium.Icon(color='green')).add_to(out_map)
    return out_map

markers = add_station_markers(base, merge_live_pred_df)
markers 
!ls<jupyter_output>[34mData[m[m                             capstone_TimeSeries.ipynb
LICENSE                          capstone_TimeSeries_hourly.ipynb
Live_station_status.ipynb        functions.py
README.md                        markers.html
TimeSeries_BikeStation.ipynb     station_pred_df.pickle
capstone_EDA.ipynb               station_pred_df1.pickle
capstone_TimeSeries-Copy1.ipynb<jupyter_text>Create dictionary key as station name and values as lat & long. The purpose is for flask app dropdown in index page.<jupyter_code>live_station_df.groupby('name')['latitude','longitude'].apply(lambda x: x.values.tolist()).to_dict()
d = {}
for i in df['name'].unique():
    d[i] = [{live_station_df['name'][j]: (live_station_df['latitude'][j], live_station_df['longitude'][j])} for j in 
            live_station_df[live_station_df['name']==i].index]
d<jupyter_output><empty_output><jupyter_text>Take station, name, latitude, longitude features and save it as csv or json format<jupyter_code>station_name_df = merge_live_pred_df[['station', 'name', 'latitude', 'longitude']]
station_name_df.shape
station_name_df.head()
station_name_df.to_csv("station_name.csv")
import csv

with open('station_name.csv') as csvfile:
        reader = csv.DictReader(csvfile)
        stations = {}
        for row in reader:
            kiosk_id= row['station']
            lat = row['latitude']
            lng = row['longitude']
            stations[kiosk_id] = (lat, lng)

import json

with open('station_name.json', 'w') as outfile:
    json.dump(stations, outfile)
!ls<jupyter_output>[34mData[m[m                             df.csv
LICENSE                          df_9_26_6pm.csv
Live_station_status.ipynb        df_9_30_5pm.csv
README.md                        functions.py
TimeSeries_BikeStation.ipynb     markers.html
capstone_EDA.ipynb               station_name.csv
capstone_TimeSeries-Copy1.ipynb  station_name.json
capstone_TimeSeries.ipynb        station_pred_df.pickle
capstone_TimeSeries_hourly.ipynb station_pred_df1.pickle
{'fraction_non_alphanumeric': 0.12293709638842382, 'fraction_numerical': 0.01064338674958144, 'mean_word_length': 3.856562137049942, 'pattern_counts': {'":': 0, '<': 46, '<?xml version=': 0, '>': 57, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 3, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '9', 'dir': 'jupyter-structured-clean-dedup', 'id': '547506', 'license': 'MIT', 'n_tokens_mistral': 2939, 'n_tokens_neox': 2864, 'n_words': 485, 'path': 'Live_station_status.ipynb', 'repo_name': 'Jkim516/Bber', 'size': '8362'}