<jupyter_start><jupyter_text>- Creating the structure<jupyter_code># We want the 14 dats between June 16 and June 29 to be the week pair -1
print((pd.Series(['2018-06-30 00:00:01']).astype('datetime64[ns]').dt.dayofyear + 1) // 14 - 13)

print((pd.Series(['2018-06-29 00:00:01']).astype('datetime64[ns]').dt.dayofyear + 1) // 14 - 13)

print((pd.Series(['2018-06-16 00:00:01']).astype('datetime64[ns]').dt.dayofyear + 1) // 14 - 13)

print((pd.Series(['2018-06-15 00:00:01']).astype('datetime64[ns]').dt.dayofyear + 1) // 14 - 13)
df = orders.copy()
df['weekpair'] = (df.time.dt.dayofyear + 1) // 14 - 13
df.weekpair.max(), df.weekpair.min()
npairs = df.weekpair.nunique()
n_items = items['itemID'].nunique()
print('total number of items:', n_items)
print('expected number of instances:', n_items * npairs)
mi = pd.MultiIndex.from_product([range(-npairs, 0), items['itemID']], names=['weekpair', 'itemID'])
data = pd.DataFrame(index = mi)
data = data.join(df.groupby(['weekpair', 'itemID'])[['order']].sum(), how = 'left')
data.fillna(0, inplace = True)
data.groupby('itemID').count().min()
data
aux = data.copy()<jupyter_output><empty_output><jupyter_text>- Creating features<jupyter_code>data = data.reset_index()
items.head(2)
data = pd.merge(data, items[['itemID', 'manufacturer', 'category1', 'category2', 'category3']], on = 'itemID')
df.head(2)
data.sort_values('weekpair', inplace = True)
data
# features = [
#     ('itemID', 'item'),
#     ('manufacturer', 'manuf'),
#     ('category1', 'cat1'),
#     ('category2', 'cat2'),
#     ('category3', 'cat3')
# ]
# for f, n in features:
#     if f not in data.columns:
#         print('ops', f)
features = [('itemID', 'item')]
# f, name = ('manufacturer', 'manuf')
for f, name in features:
    print(f)
    temp = data.groupby([f, 'weekpair'])[['order']].sum()
    shifted = temp.groupby(f)[['order']].shift(1)
    new_feature_block = pd.DataFrame()
    for n in range(3):
        rolled = shifted.groupby(f, as_index = False)['order'].rolling(2 ** n).mean()
        new_feature_block['%s_%d' % (name, 2 ** n)] = rolled.reset_index(0, drop = True) # rolling has a weird index behavior...
    data = pd.merge(data, new_feature_block.reset_index(), on = [f, 'weekpair'])
data.count() # the larger the window, more NaN are expected
data.fillna(-999, inplace=True)
# checking if we got what we wanted
data.query('itemID == 1')
# data['weekswithtrans'] = data.groupby('itemID')['order'].apply(lambda x : (x > 0).cumsum()) / (data['weekpair'] + 14)
counts = data.groupby('itemID')['order'].apply(lambda x : (x > 0).sum())
plt.hist(counts, bins = range(17))
plt.ylabel('quantidade de itens')
plt.xlabel('semanas com transação')
plt.show()
# f = 'itemID'
# temp = data.groupby([f, 'week'])[['order']].sum()
# shifted = temp.groupby(f)[['order']].shift(1)
# data['feature'] = (shifted.fillna(0) > 0).groupby(f)[['order']].cumsum().values
data.head()<jupyter_output><empty_output><jupyter_text>- Split Data<jupyter_code>filtered_data = data
sub_week = -1
train = filtered_data.query('-13 <= weekpair <= (@sub_week - 2)').reset_index(drop = True)
val = filtered_data.query('weekpair == (@sub_week - 1)').reset_index(drop = True)
sub = filtered_data.query('weekpair == (@sub_week)').reset_index(drop = True)
len(train), len(val), len(sub)
weights = infos.set_index('itemID')['simulationPrice'].to_dict()
train.columns
y_train = train.pop('order').values
y_val = val.pop('order').values
y_sub = sub.pop('order').values

X_train = train.values
X_val = val.values
X_sub = sub.values

w_train = train['itemID'].map(weights)
w_val = val['itemID'].map(weights)
w_sub = sub['itemID'].map(weights)<jupyter_output><empty_output><jupyter_text>- Min Expected Error<jupyter_code>def evaluate(prediction, target, simulationPrice):
    return np.sum((prediction - np.maximum(prediction - target, 0) * 1.6)  * simulationPrice)
# max expected rmse
from sklearn.metrics import mean_squared_error as mse
# pred = data.loc[1:12].groupby('itemID')['order'].mean().sort_index()
# target_week = data.loc[13:, 'order'].reset_index(level = 0, drop = True).sort_index()
# mse(target_week, pred) ** .5<jupyter_output><empty_output><jupyter_text>- XGBoost<jupyter_code>import xgboost as xgb
# custom objective

def gradient(prediction, dtrain):
    y = dtrain.get_label()
#     prediction.astype(int)
#     prediction = np.minimum(prediction.astype(int), 1)
    return -2 * (prediction - np.maximum(prediction - y, 0) * 1.6) * (1 - (prediction > y) * 1.6)

def hessian(prediction, dtrain):
    y = dtrain.get_label()
#     prediction.prediction(int)
#     prediction = np.minimum(prediction.astype(int), 1)
    return -2 * (1 - (prediction > y) * 1.6) ** 2

def objective(prediction, dtrain):
    w = dtrain.get_weight()
    grad = gradient(prediction, dtrain) * w
    hess = hessian(prediction, dtrain) * w
    return grad, hess
# custom feval

def feval(prediction, dtrain):
    prediction = prediction.astype(int)
#     predt = np.minimum(predt.astype(int), 1)
    target = dtrain.get_label()
    simulationPrice = dtrain.get_weight()
    return 'feval', np.sum((prediction - np.maximum(prediction - target, 0) * 1.6)  * simulationPrice)
dtrain = xgb.DMatrix(X_train, y_train, w_train, missing = -999)
dval = xgb.DMatrix(X_val, y_val, w_val, missing = -999)
dsub = xgb.DMatrix(X_sub, y_sub, w_sub, missing = -999)
# specify parameters via map
param = {
    'max_depth':6,
    'eta':0.005,
    'objective':'reg:squarederror',
    'disable_default_eval_metric': 1
}
num_round = 100
bst = xgb.train(param, dtrain,
                num_round, early_stopping_rounds = 5,
#                 obj = objective,
                feval = feval, maximize = True,
                evals = [(dtrain, 'train'), (dval, 'val')])
prediction = bst.predict(dsub, ntree_limit=bst.best_ntree_limit).astype(int)
evaluate(prediction, y_sub, w_sub)
# max score
evaluate(y_sub, y_sub, w_sub)
# using previous weekpair
evaluate(y_val, y_sub, w_sub)
submission = items[['itemID']].copy()
submission['demandPrediction'] = bst.predict(dsub, ntree_limit=bst.best_ntree_limit).astype(int)
submission.to_csv('../../submissions/sub_inclass_03.csv', sep = '|', index=False)
# submission.head()<jupyter_output><empty_output><jupyter_text>- LGBM<jupyter_code>def feval_lgbm(prediction, dtrain):
    prediction = prediction.astype(int)
    target = dtrain.get_label()
    simulationPrice = dtrain.get_weight()
    return 'feval', np.sum((prediction - np.maximum(prediction - target, 0) * 1.6)  * simulationPrice), True
import lightgbm as lgb

params = {
          "objective" : 'regression_l1',
#           "metric" :"rmse",
#           "learning_rate" : 0.01,
          'verbosity': 2,
#           'max_depth': 6,
#           'num_leaves': 15,
          "min_data_in_leaf":3000
         }
# https://lightgbm.readthedocs.io/en/latest/Parameters.html


ds_params = {
#     'categorical_feature' : [8, 9, 10],
}
lgbtrain = lgb.Dataset(X_train, label = y_train, weight=w_train, **ds_params)
lgbvalid = lgb.Dataset(X_val, label = y_val, weight=w_val, **ds_params)
lgbsubmis = lgb.Dataset(X_sub, label = y_sub, weight=w_sub, **ds_params)

num_round = 1000
lgb_model = lgb.train(params,
                  lgbtrain,
                  num_round,
                  valid_sets = [lgbtrain, lgbvalid],
                  valid_names = ['train', 'val'],
                  verbose_eval=5,
                  early_stopping_rounds=5,
                 feval = feval_lgbm,
#                   fobj = objective,
                 )
prediction = lgb_model.predict(X_sub, num_iteration=lgb_model.best_iteration).astype(int)
evaluate(prediction, y_sub, w_sub)<jupyter_output><empty_output><jupyter_text>- CatBoost<jupyter_code>from catboost import CatBoost, CatBoostRegressor, Pool
class feval_cat(object):
    def get_final_error(self, error, weight):
        # return error / (weight + 1e-38)
        return error

    def is_max_optimal(self):
        return True

    def evaluate(self, approxes, target, simulationPrice):
        prediction = np.array(approxes[0]).astype(int)
        target = np.array(target[0]).astype(int)
        simulationPrice = np.array(simulationPrice[0])
        score = np.sum((prediction - np.maximum(prediction - target, 0) * 1.6)  * simulationPrice)
#         print('score', score)
#         print(type(prediction), type(target), type(simulationPrice))
        return np.sum((prediction - np.maximum(prediction - target, 0) * 1.6)  * simulationPrice), 0
ds_params = {
#     'cat_features' : [8, 9, 10],
}
train_pool = Pool(X_train, label = y_train, weight = w_train, **ds_params)
val_pool = Pool(X_val, label = y_val, weight = w_sub, **ds_params)
sub_pool = Pool(X_sub, label = y_sub, weight = w_sub, **ds_params)


model = CatBoostRegressor(
#     iterations = 2,
#     depth=6, 
#     learning_rate=0.1, 
    loss_function='MAE',
    early_stopping_rounds=5,
    eval_metric = feval_cat(),
    thread_count=-1,
)

model.fit(
    train_pool,
    eval_set=[train_pool, val_pool],
#     logging_level='Verbose',  # you can uncomment this for text output

);

prediction = model.predict(X_val, ntree_end = model.best_iteration_).astype(int)
evaluate(prediction, y_val, w_val)
prediction = model.predict(X_sub, ntree_end = model.best_iteration_).astype(int)
evaluate(prediction, y_sub, w_sub)<jupyter_output><empty_output><jupyter_text>- Ensemble<jupyter_code>cat_w = 1
lgb_w = 1
xgb_w = 1
ensemble = model.predict(X_sub, ntree_end = model.best_iteration_) * cat_w
ensemble += lgb_model.predict(X_sub, num_iteration=lgb_model.best_iteration) * lgb_w
ensemble += bst.predict(dsub, ntree_limit=bst.best_ntree_limit) * xgb_w
ensemble = ensemble / (cat_w + lgb_w)
evaluate(ensemble.astype(int), y_sub, w_sub)<jupyter_output><empty_output><jupyter_text>- Linear Regression<jupyter_code>from sklearn.linear_model import LinearRegression
# from sklearn.metrics import 
lr = LinearRegression()
lr.fit(X_train, y_train, w_train)
print('train', evaluate(lr.predict(X_train), y_train, w_train))
print('test', evaluate(lr.predict(X_test), y_test, w_test))
print('sub', evaluate(lr.predict(X_sub), y_sub, w_sub))<jupyter_output>train -21058674.042366117
test -2602271.639547858
sub -2761693.8996196142<jupyter_text><jupyter_code># fazer feature que pega o percentil de quando o item deu de dinheiro dentro de uma categoria
# fazer features que pega dist de atual até o pico mais alto
# dist do maior pico pro segundo
# min(dist(terceiro, primeiro), dist(terceiro, segundo))<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.16077907198777927, 'fraction_numerical': 0.02692381134237159, 'mean_word_length': 4.338939857288481, 'pattern_counts': {'":': 1, '<': 42, '<?xml version=': 0, '>': 45, 'https://': 1, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '10', 'dir': 'jupyter-structured-clean-dedup', 'id': '171289', 'license': 'MIT', 'n_tokens_mistral': 4064, 'n_tokens_neox': 3821, 'n_words': 753, 'path': 'tobias/models.ipynb', 'repo_name': 'joaopedromattos/DMC2020', 'size': '10474'}