<jupyter_start><jupyter_text>Prepare the Y Variable<jupyter_code>orders.set_index('order_id', inplace = True, drop = False)
train.set_index('order_id', inplace = True, drop = False)
train_eval.set_index('order_id', inplace = True, drop = False)




train = train.join(orders, on = 'order_id', rsuffix = '_')
train.drop('order_id_', inplace = True, axis = 1)

train_eval = train_eval.join(orders, on = 'order_id', rsuffix = '_')
train_eval.drop('order_id_', inplace = True, axis = 1)



train.reset_index(inplace=True, drop=True)
train_eval.reset_index(inplace=True, drop=True)
orders.reset_index(inplace=True, drop=True)


y_df = pd.DataFrame()
y_df['y'] = train.groupby('user_id')['product_id'].agg('count')

y_df.reset_index(inplace = True, drop = False)

y_df.head()
y_eval_df = pd.DataFrame()
y_eval_df['y'] = train_eval.groupby('user_id')['product_id'].agg('count')

y_eval_df.reset_index(inplace = True, drop = False)

y_eval_df.head()
print('Join orders with prior')
orders.set_index('order_id', inplace = True, drop = False)
priors.set_index('order_id', inplace = True, drop = False)

priors = priors.join(orders, on = 'order_id', rsuffix = '_')
priors.drop('order_id_', inplace = True, axis = 1)

priors.reset_index(inplace=True, drop=True)
orders.reset_index(inplace=True, drop=True)


print('Prepare prior for featrues')

prior_subset = pd.DataFrame()

prior_subset['p_count']    = priors.groupby(['user_id','order_id','order_number'], group_keys = False)['product_id'].agg('count')
prior_subset['rorder_sum'] = priors.groupby(['user_id','order_id','order_number'], group_keys = False)['reordered'].agg('sum')
prior_subset['rorder_rate'] = prior_subset['rorder_sum'] / prior_subset['p_count']




prior_subset.reset_index(drop = False, inplace=True)
prior_subset.sort_values(by = ['user_id','order_number'], inplace = True, ascending = [True, True], axis = 0)
prior_subset.drop('order_id', inplace = True, axis = 1)


prior_subset.reset_index(inplace = True, drop = True)

prior_subset.head()<jupyter_output><empty_output><jupyter_text>Run timeseries_features.py<jupyter_code>ts_features = pd.read_csv('./features/ts_features.csv')
ts_features.head()
ts_features['user_id'] = prior_subset['user_id'].unique()
ts_features.head(10)



y_df = y_df.join(ts_features, on='user_id', rsuffix='_')
y_df.drop(['user_id_'], inplace = True, axis = 1)


y_eval_df = y_eval_df.join(ts_features, on='user_id', rsuffix='_')
y_eval_df.drop(['user_id_'], inplace = True, axis = 1)

## Replace na with 0
nan_rows = y_df[y_df.isnull().T.any().T]
y_df.fillna(value = 0, inplace = True)

## Replace na with 0
nan_rows = y_eval_df[y_eval_df.isnull().T.any().T]
y_eval_df.fillna(value = 0, inplace = True)



y_df.to_csv('./features/regression/y_df.csv', index = False)
y_eval_df.to_csv('./features/regression/y_eval_df.csv', index = False)


y_df.head()
y_eval_df.head()<jupyter_output><empty_output><jupyter_text>Build the model<jupyter_code>import lightgbm as lgb

train   = lgb.Dataset(y_df.ix[:,2:], y_df.ix[:,1])
evalset = lgb.Dataset(y_eval_df.ix[:,2:], y_eval_df.ix[:,1])

# specify your configurations as a dict
params = {
    'task': 'train',
    'boosting_type': 'gbdt',
    'objective': 'poisson',
    'metric': {'poisson', 'mse'},
    'num_leaves': 31,
    'learning_rate': 0.1,
    'feature_fraction': 1.0,
    'bagging_fraction': 0.9,
    'bagging_freq': 10,
    'verbose': 0
}

print('Start training...')
# train
gbm = lgb.train(params,
                train,
                num_boost_round=20,
                valid_sets=evalset,
                early_stopping_rounds=5)

print('Save model...')
# save model to file
gbm.save_model('./models/gbm_reg_model.txt')



from sklearn.metrics import mean_squared_error
print('Start predicting...')
# predict
y_pred = gbm.predict(y_eval_df.ix[:,2:], num_iteration=gbm.best_iteration)
# eval
print('The rmse of prediction is:', mean_squared_error(y_eval_df.ix[:,1], y_pred) ** 0.5)
y_pred
# feature importances

for i,(n,s) in enumerate(zip(train.feature_name, list(gbm.feature_importance()))):
    print i,'  :',n,s

%matplotlib inline
import matplotlib.pyplot as plt


plt.scatter(y_eval_df.ix[:,9], y_eval_df.ix[:,1])
y_eval_df.ix[:,8]
from sklearn.ensemble import RandomForestRegressor


    
forest = RandomForestRegressor(n_estimators = 200, max_features = 'sqrt' ,random_state = 100, n_jobs = -1, verbose = 1, oob_score = True)
forest.fit(X.values, Y.values)
oob_p = forest.oob_prediction_
oob_p = np.round(oob_p, 0)


np.mean(np.abs(oob_p - Y.values))

oob_p


from sklearn.ensemble import RandomForestRegressor

oob_scores = []
r2_scores = []
n_trees = range(100,200,10)

for ne in n_trees:
    
    print "Starting ", ne
    forest = RandomForestRegressor(n_estimators = 200, max_features = 2 ,random_state = 100, n_jobs = -1, verbose = 0, oob_score = True)
    forest.fit(X.values, Y.values)
    oob_p = forest.oob_prediction_
    y_p = np.round(oob_p,0)
    oob_scores.append(sum(abs(y_p - Y.values)) / ( 1.0 * len(y_p)))
    r2_scores.append(forest.score(X.values, Y.values))


%matplotlib inline


import  matplotlib.pyplot as plt

plt.scatter(n_trees, oob_scores)

plt.scatter(n_trees, r2_scores)


X_eval = y_eval_df.ix[:,2:]
Y_eval = y_eval_df.ix[:,1]

## Dummy variables
X_eval = pd.get_dummies(X_eval, prefix = ['o1_order_dow','o2_order_dow','o3_order_dow'
                                             ,'o1_order_hr','o2_order_hr','o3_order_hr'],
                   columns = ['o1_order_dow','o2_order_dow','o3_order_dow'
                                             ,'o1_order_hr','o2_order_hr','o3_order_hr'])

#forest.score(X_eval.values, Y_eval.values)
y_p = forest.predict(X_eval.values)
y_p = np.round(y_p,0)

np.mean(np.abs(Y_eval.values - y_p))

Y_eval.values<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.15879902811523777, 'fraction_numerical': 0.01475182228392919, 'mean_word_length': 4.850761421319797, 'pattern_counts': {'":': 0, '<': 13, '<?xml version=': 0, '>': 13, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '3', 'dir': 'jupyter-structured-clean-dedup', 'id': '154990', 'license': 'Apache-2.0', 'n_tokens_mistral': 2361, 'n_tokens_neox': 2253, 'n_words': 375, 'path': 'Model-Order-Size-Regression.ipynb', 'repo_name': 'subramgo/Instacart', 'size': '5762'}