<jupyter_start><jupyter_text>Read data<jupyter_code>data = {
    'av': pd.read_csv('data/air_visit_data.csv'),
    'as': pd.read_csv('data/air_store_info.csv'),
    'hs': pd.read_csv('data/hpg_store_info.csv'),
    'ar': pd.read_csv('data/air_reserve.csv'),
    'hr': pd.read_csv('data/hpg_reserve.csv'),
    'sr': pd.read_csv('data/store_id_relation.csv'),
    'ss': pd.read_csv('data/sample_submission.csv'),
    'di': pd.read_csv('data/date_info.csv', parse_dates=True).rename(columns={'calendar_date':'visit_date'})
    }<jupyter_output><empty_output><jupyter_text>Prepare date_info<jupyter_code>data['di'].head()
data['di']['visit_date'] = pd.to_datetime(data['di']['visit_date'])
#data['di'].set_index(['visit_date'], inplace=True)
#data['di'].index.name = None
data['di'].day_of_week = data['di'].day_of_week.apply(list(calendar.day_name).index)
data['di'].head()<jupyter_output><empty_output><jupyter_text>Prepare air_visit_data<jupyter_code>data['av'].head()
data['av']['visit_date'] = pd.to_datetime(data['av']['visit_date'])
data['av'] = pd.merge(data['av'], data['di'], how='left', on=['visit_date'])
data['av'].head()
data['av']['day_of_week'] = data['av'].apply(lambda row: 7 if row['holiday_flg'] == 1 else row['day_of_week'], axis=1)
data['av'].drop(['holiday_flg'], axis=1, inplace=True)
data['av'].head()<jupyter_output><empty_output><jupyter_text>Prepare submission data<jupyter_code>data['ss'].head()
data['ss']['store_id'] = data['ss'].apply(lambda row: row.loc['id'][:row.loc['id'].rindex('_')], axis=1)
data['ss']['visit_date'] = data['ss'].apply(lambda row: row.loc['id'][row.loc['id'].rindex('_')+1:], axis=1)
data['ss']['visit_date'] = pd.to_datetime(data['ss']['visit_date'])
data['ss'].head()
data['ss'] = pd.merge(data['ss'], data['di'], how='left', on=['visit_date'])
data['ss'].head()
data['ss']['day_of_week'] = data['ss'].apply(lambda row: 7 if row['holiday_flg'] == 1 else row['day_of_week'], axis=1)
data['ss'].drop(['holiday_flg'], axis=1, inplace=True)
data['ss'].head()
def visualize():
    ts_train = data['av'].groupby(['visit_date'])[['visitors']].sum()
    ts_pred = data['ss'].groupby(['visit_date'])[['visitors']].sum()
    plt.rcParams["figure.figsize"] = (16,4)
    plt.plot(ts_train)
    plt.plot(ts_pred)
    plt.show()

    ts_train = data['av'][:391][['visit_date', 'visitors']].set_index('visit_date')
    ts_pred = data['ss'][23361:23399][['visit_date', 'visitors']].set_index('visit_date')
    plt.plot(ts_train)
    plt.plot(ts_pred)
    plt.show()
def RMSLE(y_pred, y_test):   
    return mean_squared_log_error(y_test, y_pred)**0.5<jupyter_output><empty_output><jupyter_text>Prediction based on median for all days<jupyter_code>medians = data['av'].groupby('air_store_id').median()
medians_by_days = data['av'].groupby(['air_store_id', 'day_of_week']).median()
general_median = medians['visitors'].median()
medians_by_days[:10][:]
def predict_by_median(row):
    store_id = row.loc['store_id']
    row.loc['visitors'] = medians.loc[store_id]['visitors'] if store_id in medians.index else general_median
    return row
data['ss'] = data['ss'].apply(predict_by_median, axis=1)
data['ss'].head()
visualize()
data['ss'].to_csv('data/prediction_median.csv', index=False, columns=['id', 'visitors'])<jupyter_output><empty_output><jupyter_text>Prediction based on median for particular day of week (with festivals as 8th day)<jupyter_code>medians_by_days = data['av'].groupby(['air_store_id', 'day_of_week']).median()
general_median_for_day = data['av'].groupby('day_of_week').median()
medians_by_days[:10][:]
def predict_by_median_for_concrete_day(row):
    store_id = row.loc['store_id']
    day_of_week = row.loc['day_of_week']
    if store_id in medians_by_days.index:
        if day_of_week in medians_by_days.loc[store_id].index:
            row.loc['visitors'] = medians_by_days.loc[store_id, day_of_week]['visitors']
        else:
            row.loc['visitors'] = medians_by_days.loc[store_id].values.mean()
    else:
        row.loc['visitors'] = general_median_for_day.loc[day_of_week]['visitors']
        
    return row
data['ss'] = data['ss'].apply(predict_by_median_for_concrete_day, axis=1)
data['ss'].head()
visualize()
data['ss'].to_csv('data/prediction_median_for_concrete_day.csv', index=False, columns=['id', 'visitors'])<jupyter_output><empty_output><jupyter_text>Prediction based on proportinal dependency for particular day of week (with festivals as 8th day)<jupyter_code>groupby_store_day = data['av'].groupby(['air_store_id', 'day_of_week'])
groupby_store = data['av'].groupby(['air_store_id'])
def exponential_smoothing(series, alpha):
    result = [series[0]] # first value is same as series
    for n in range(1, len(series)):
        result.append(alpha * series[n] + (1 - alpha) * result[n-1])
    return np.median(result)
pred_values_dict = {}
def predict_by_proportional_for_concrete_day2(row):
    store_id = row.loc['store_id']
    day_of_week = row.loc['day_of_week']
    key = (store_id, day_of_week)
    if key not in pred_values_dict:
        pred_values_dict[key] = []
    pred_values = pred_values_dict[key]     
    if (store_id, day_of_week) in groupby_store_day.groups.keys():
        train_values = groupby_store_day.get_group((store_id, day_of_week))['visitors'].values
    else:
        train_values = groupby_store.get_group(store_id)['visitors'].values
    
    values = np.hstack((train_values, pred_values))
    pred = exponential_smoothing(values[::-1], 0.25)        
    pred_values_dict[key].append(pred)
    row.loc['visitors'] = pred
    return row
data['ss'] = data['ss'].apply(predict_by_proportional_for_concrete_day2, axis=1)
data['ss'].head()
visualize()
data['ss'].to_csv('data/prediction_proportional_025_for_concrete_day.csv', index=False, columns=['id', 'visitors'])<jupyter_output><empty_output><jupyter_text>Experiments<jupyter_code>data['av'].groupby(['air_store_id', 'day_of_week']).groups
for name,group in data['av'].groupby(['air_store_id', 'day_of_week']):
    print name
    print group
data['av'].groupby(['air_store_id', 'day_of_week']).get_group(('air_ba937bf13d40fb24', 7))['visitors'].values
data['av'].groupby(['air_store_id', 'day_of_week']).last().loc['air_ba937bf13d40fb24', 7]
df_day = data['av'].groupby(['air_store_id', 'day_of_week']).get_group(('air_ba937bf13d40fb24', 0))[['visit_date', 'visitors']]
df_day.set_index(['visit_date'], inplace=True)
df_day.index.name = None
df_day.head()
df_day.plot( figsize=(16,4))
df_all = data['av'].groupby(['air_store_id']).get_group('air_ba937bf13d40fb24')[['visit_date', 'visitors']]
df_all.set_index(['visit_date'], inplace=True)
df_all.index.name = None
df_all.head()
df_all.plot( figsize=(16,4))<jupyter_output><empty_output><jupyter_text>Prepare air_reserve<jupyter_code>data['ar']['visit_datetime'] = pd.to_datetime(data['ar']['visit_datetime'])
data['ar']['visit_date'] = pd.to_datetime(data['ar']['visit_datetime'].dt.date)
data['ar']['reserve_datetime'] = pd.to_datetime(data['ar']['reserve_datetime'])
data['ar']['reserve_date'] = pd.to_datetime(data['ar']['reserve_datetime'].dt.date)
data['ar']['reserve_delta'] = (data['ar']['visit_datetime']-data['ar']['reserve_datetime']).astype('timedelta64[h]')
data['ar'].drop(['visit_datetime', 'reserve_datetime'], inplace=True, axis=1)
data['ar'].head()
len(data['ar']['air_store_id'].unique())
air_reservations_by_dates = data['ar'].groupby(['air_store_id', 'visit_date'])[['reserve_visitors']].sum()
air_reservations_by_dates.head()<jupyter_output><empty_output><jupyter_text>Prepare hpg_reserve<jupyter_code>data['hr']['visit_datetime'] = pd.to_datetime(data['hr']['visit_datetime'])
data['hr']['visit_date'] = pd.to_datetime(data['hr']['visit_datetime'].dt.date)
data['hr']['reserve_datetime'] = pd.to_datetime(data['hr']['reserve_datetime'])
data['hr']['reserve_date'] = pd.to_datetime(data['hr']['reserve_datetime'].dt.date)
data['hr']['reserve_delta'] = (data['hr']['visit_datetime']-data['hr']['reserve_datetime']).astype('timedelta64[h]')
data['hr'].drop(['visit_datetime', 'reserve_datetime'], inplace=True, axis=1)
data['hr'].head()
hpg_reservations_by_dates = data['hr'].groupby(['hpg_store_id', 'visit_date'])[['reserve_visitors']].sum()
hpg_reservations_by_dates.head()
len(data['hr']['hpg_store_id'].unique())<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.20516533912623702, 'fraction_numerical': 0.012309920347574221, 'mean_word_length': 7.949244060475162, 'pattern_counts': {'":': 0, '<': 41, '<?xml version=': 0, '>': 41, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '10', 'dir': 'jupyter-structured-clean-dedup', 'id': '257196', 'license': 'MIT', 'n_tokens_mistral': 3179, 'n_tokens_neox': 3098, 'n_words': 370, 'path': 'Restaurant forecasting baseline.ipynb', 'repo_name': 'astarostin/Recruit-Restaurant-Visitor-Forecasting', 'size': '8286'}