<jupyter_start><jupyter_text>Path and other variables<jupyter_code>path = Path('/kaggle/m5_forecasting/')
assert(path.exists())
h = 28 
max_lags = h * 2 + 1
tr_last = 1913
fday = datetime(2016, 4, 25)
cal = pd.read_csv(path/"calendar.csv", dtype = m5_common.CAL_DTYPES)
%%time

prices, cal = m5_common.prepare_tables(path)
cal
prices
event_name_1_map, event_type_1_map = m5_common.replace_cal_cols(cal)
uint8_types= ['event_name_1', 'event_type_1', 'event_name_2', 'event_type_2', 'month', 'wday', 'weekday', 
              'snap_CA', 'snap_TX', 'snap_WI']
m5_common.convert_uint8(cal, uint8_types)
m5_common.add_days_before(cal)
FIRST_DAY = 1 # If you want to load all the data set it to '1' -->  Great  memory overflow  risk !
%%time

df = m5_common.create_dt(cal, prices, is_train=True, first_day=FIRST_DAY, tr_last=tr_last, path=path)
df.info()
df
def extract_by_state(dt):
    df_TX = dt[dt['state_id'] == m5_common.state_map['TX']]
    df_WI = dt[dt['state_id'] == m5_common.state_map['WI']]
    df_CA = dt[dt['state_id'] == m5_common.state_map['CA']]
    del df_TX['snap_CA']
    del df_TX['snap_WI']
    del df_CA['snap_TX']
    del df_CA['snap_WI']
    del df_WI['snap_TX']
    del df_WI['snap_CA']
    df_TX.rename(columns={'snap_TX': 'snap'}, inplace=True)
    df_CA.rename(columns={'snap_CA': 'snap'}, inplace=True)
    df_WI.rename(columns={'snap_WI': 'snap'}, inplace=True)
    return df_TX, df_WI, df_CA
%%time

df_TX, df_WI, df_CA = extract_by_state(df)
df_TX.shape, df_WI.shape, df_CA.shape
assert len(df_TX) + len(df_WI) + len(df_CA) == len(df)
del df
gc.collect()
df_CA.info()
def create_fea(dt):
    
    grouped_sales = dt[["id","sales"]].groupby("id")["sales"]
    
    wins = [7, 28]
    lags = [7, 28]
    
    for win in wins:
        mean_col = f'mean_{win}'
        emean_col = f'e{mean_col}' # exponential weighted average
        esmean_col = f'es{mean_col}'
        dt[emean_col] = grouped_sales.transform(lambda x : x.ewm(span=win, adjust=False).mean())
        dt[esmean_col] = grouped_sales.transform(lambda x : x.ewm(alpha=1/win, adjust=False).mean())
        for lag in lags:
            dt[f'emean_{win}_{lag}'] = dt[["id", emean_col]].groupby("id").shift(lag)
            dt[f'esmean_{win}_{lag}'] = dt[["id", esmean_col]].groupby("id").shift(lag)
        del dt[emean_col]
        del dt[esmean_col]
            
    ra = [1, 2]
    for simple_lag in ra:
        dt[f'lag_{simple_lag}'] = grouped_sales.shift(simple_lag)
%%time

for df in [df_CA, df_TX, df_WI]:
    create_fea(df)
df_CA.info()
df_WI
df_TX
%%time

for df in [df_CA, df_TX, df_WI]:
    df.dropna(inplace = True)
fig, ax = plt.subplots(figsize=(8, 6))
sns.distplot(df_CA['sales'][:300000])
fig, ax = plt.subplots(figsize=(8, 6))
sns.distplot(df_WI['sales'][:300000])
fig, ax = plt.subplots(figsize=(8, 6))
sns.distplot(df_TX['sales'][:300000])
leave_size = 10
n_estimators = 1300

params_CA = {
        "objective" : "tweedie",
        'tweedie_variance_power': 1.075,
        "metric" :"rmse",
#         "force_row_wise" : True,
        "learning_rate" : 0.075,
#         "sub_feature" : 0.8,
        "sub_row" : 0.75,
        "bagging_freq" : 1,
        "feature_fraction": 1.0,
        "lambda_l2" : 0.1,
        "nthread" : 10,
        "metric": ["rmse"],
        'verbosity': 20,
        'num_leaves': 2**leave_size-1,
        "min_data_in_leaf": 2**(leave_size + 1)-1,
        "n_estimators": n_estimators
}

params_TX = {
        "objective" : "tweedie",
        'tweedie_variance_power': 1.2,
        "metric" :"rmse",
#         "force_row_wise" : True,
        "learning_rate" : 0.075,
#         "sub_feature" : 0.8,
        "sub_row" : 0.75,
        "bagging_freq" : 1,
        "feature_fraction": 1.0,
        "lambda_l2" : 0.1,
        "nthread" : 10,
        "metric": ["rmse"],
        'verbosity': 20,
        'num_leaves': 2**leave_size-1,
        "min_data_in_leaf": 2**(leave_size + 1)-1,
        "n_estimators": n_estimators
}

params_WI = {
        "objective" : "tweedie",
        'tweedie_variance_power': 1.1,
        "metric" :"rmse",
#         "force_row_wise" : True,
        "learning_rate" : 0.07,
#         "sub_feature" : 0.8,
        "sub_row" : 0.75,
        "bagging_freq" : 1,
        "feature_fraction": 1.0,
        "lambda_l2" : 0.1,
        "nthread" : 10,
        "metric": ["rmse"],
        'verbosity': 20,
        'num_leaves': 2**leave_size-1,
        "min_data_in_leaf": 2**(leave_size + 1)-1,
        "n_estimators": n_estimators
}
np.random.seed(777)

cat_feats = ['item_id', 'dept_id','store_id', 'cat_id', 'state_id', "event_name_1", "event_name_2", "event_type_1",  "event_type_2", 'snap']
useless_cols = ["id", "date", "sales","d", "wm_yr_wk", "weekday", "revenue"]

for name, df, params in zip(['CA', 'TX', 'WI'], [df_CA, df_TX, df_WI], [params_CA, params_TX, params_WI]):
    
    train_cols = df.columns[~df.columns.isin(useless_cols)]
    X_train = df[train_cols]
    y_train = df["sales"]

    size_valid_set = len(df) // 10
    fake_valid_inds = np.random.choice(X_train.index.values, size_valid_set, replace = False)
    train_inds = np.setdiff1d(X_train.index.values, fake_valid_inds)
    
    X = X_train.loc[train_inds]
    y = y_train.loc[train_inds]

    print(f'Starting training {name} {params["tweedie_variance_power"]}')
    
    m_lgb = lgb.LGBMRegressor(**params)
    m_lgb.fit(X=X, y=y, 
              eval_set=[(X, y), (X_train.loc[fake_valid_inds], y_train.loc[fake_valid_inds])],
              eval_names=['train sales', 'valid sales'], 
              eval_metric=params['metric'],
              verbose=params['verbosity'],
              early_stopping_rounds=100,
              categorical_feature=cat_feats)
    m_lgb.booster_.save_model(str(path/f"m5_model_{name}.lgb"))
!tar -czvf {path/'m5_models.lgb'} {path/'m5_model_WI.lgb'} {path/'m5_model_TX.lgb'} {path/'m5_model_CA.lgb.tgz'}<jupyter_output>tar: Removing leading `/' from member names
/kaggle/m5_forecasting/m5_model_WI.lgb
/kaggle/m5_forecasting/m5_model_TX.lgb
/kaggle/m5_forecasting/m5_model_CA.lgb<jupyter_text>Predict<jupyter_code>%%time

from tqdm.notebook import tqdm

sub = 0.
cols = [f"F{i}" for i in range(1,29)]
df = m5_common.create_dt(cal, prices, False, first_day=FIRST_DAY, path=path)
df_TX, df_WI, df_CA = extract_by_state(df)
max_lags = h * 4 + 1

all_results = []
for name, te in zip(['CA', 'TX', 'WI'], [df_CA, df_TX, df_WI]):
    m_lgb = lgb.Booster(model_file=str(path/f"m5_model_{name}.lgb"))
    for tdelta in tqdm(range(0, h), total=h):
        day = fday + timedelta(days=tdelta)
        print(name, tdelta, day)
        tst = te[(te.date >= day - timedelta(days=max_lags)) & (te.date <= day)].copy()
        create_fea(tst)
        tst = tst.loc[tst.date == day, train_cols]
        te.loc[te.date == day, "sales"] = m_lgb.predict(tst)
        
    te_sub = te.loc[te.date >= fday, ["id", "sales"]].copy()
    te_sub.loc[te.date >= fday+ timedelta(days=h), "id"] = te_sub.loc[te.date >= fday+timedelta(days=h), 
                                                                      "id"].str.replace("validation$", "evaluation")
    all_results.append(te_sub)
    
te_sub = pd.concat(all_results, axis=0, sort=False)

te_sub["F"] = [f"F{rank}" for rank in te_sub.groupby("id")["id"].cumcount()+1]
te_sub = te_sub.set_index(["id", "F" ]).unstack()["sales"][cols].reset_index()
te_sub.fillna(0., inplace = True)
te_sub.sort_values("id", inplace = True)
te_sub.reset_index(drop=True, inplace = True)
sub = te_sub

%time

sub2 = sub.copy()
sub2["id"] = sub2["id"].str.replace("validation$", "evaluation")
sub = pd.concat([sub, sub2], axis=0, sort=False)
sub.to_csv("submission.csv",index=False)
sub
!wc -l submission.csv<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.17254800207576543, 'fraction_numerical': 0.025298391281785158, 'mean_word_length': 3.367705382436261, 'pattern_counts': {'":': 12, '<': 9, '<?xml version=': 0, '>': 13, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 3, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '2', 'dir': 'jupyter-structured-clean-dedup', 'id': '215567', 'license': 'Apache-2.0', 'n_tokens_mistral': 3278, 'n_tokens_neox': 3143, 'n_words': 532, 'path': 'nbs_gil/m5/m5_lgbm_extended_3models_11.ipynb', 'repo_name': 'gilfernandes/fastai-exercises', 'size': '7708'}