<jupyter_start><jupyter_text>Library<jupyter_code>%%javascript
utils.load_extension("collapsible_headings/main")
utils.load_extension("hide_input/main")
utils.load_extension("autosavetime/main")
utils.load_extension("execute_time/ExecuteTime")
utils.load_extension("code_prettify/code_prettify")
utils.load_extension("scroll_down/main")
utils.load_extension("jupyter-js-widgets/extension")
import numpy as np
import pandas as pd

pd.set_option("display.max_columns", None)
from lightgbm import LGBMRegressor
from sklearn.linear_model import LinearRegression
from category_encoders import TargetEncoder

from sklearn.metrics import mean_absolute_error
from sklearn.pipeline import Pipeline
from sklearn.model_selection import train_test_split, cross_val_predict
from tools.metrics import (
    apply_metrics,
    prep_data_for_metric,
    get_avg_volumes,
    mean_absolute_percentage_error,
)
from catboost import CatBoostClassifier


def compute_metrics(preds, lower, upper, y, X, avg_volumes):

    id_cols = ["country", "brand"]

    prepped_X = prep_data_for_metric(X, avg_volumes)

    prepped_X["actuals"] = y
    prepped_X["forecast"] = preds
    prepped_X["lower_bound"] = lower
    prepped_X["upper_bound"] = upper

    return np.mean(abs(prepped_X.groupby(id_cols).apply(apply_metrics)))
full_df = pd.read_csv("data/gx_merged_lags.csv")
train_tuples = pd.read_csv("data/train_split.csv")
valid_tuples = pd.read_csv("data/valid_split.csv")

test_df = full_df[full_df.test == 1].copy().reset_index(drop=True)

full_df = full_df[full_df.test == 0]

train_df = full_df.merge(train_tuples, how="inner").reset_index(drop=True)
val_df = full_df.merge(valid_tuples, how="inner").reset_index(drop=True)
# TODO: no need for calculation every time
avg_volumes = get_avg_volumes()

to_drop = ["month_name", "volume"]

X_train = train_df.drop(columns=to_drop)
y_train = train_df.volume

X_val = val_df.drop(columns=to_drop)
y_val = val_df.volume

X_test = test_df.drop(columns=to_drop)<jupyter_output><empty_output><jupyter_text>Model<jupyter_code>lgb = LGBMRegressor(n_jobs=-1, n_estimators=100, objective="regression_l1")
from sktools.encoders import QuantileEncoder
categorical_cols = ["country", "brand", "therapeutic_area", "presentation"]
te = TargetEncoder(cols=categorical_cols)
te = QuantileEncoder(cols=categorical_cols)
te_residual = TargetEncoder(cols=categorical_cols)
pipe = Pipeline([("te", te), ("cb", lgb)])
cv_preds = cross_val_predict(pipe, X_train, y_train, cv=3)
pipe.fit(X_train, y_train)
preds = pipe.predict(X_val)<jupyter_output><empty_output><jupyter_text>Residuals  Predict val error<jupyter_code>lgb_residual = LGBMRegressor(n_jobs=-1, n_estimators=100, objective="regression_l1")
pipe_residual = Pipeline([("te", te_residual), ("lgb", lgb_residual)])
y_train_residual = np.abs(cv_preds - y_train)
pipe_residual.fit(X_train, y_train_residual)

preds_residual = pipe_residual.predict(X_val)<jupyter_output><empty_output><jupyter_text>Results<jupyter_code>submission_df = pd.read_csv("data/submission_template.csv")
preds_test = pipe.predict(X_test)
preds_test_residual = pipe_residual.predict(X_test)

bounds = [0, 0.01, 0.1, 0.5, 1, 1.5]


min_unc = 1e8
best_bound = 0
for bound in bounds:
    print(f"Bound: {bound}")
    metric_pair = compute_metrics(
        preds,
        preds-1,
        preds + 1,
        y_val,
        X_val,
        avg_volumes,
    )
    print(metric_pair)

    unc_metric = metric_pair.values[1]

    if unc_metric < min_unc:
        min_unc = unc_metric
        best_bound = bound
preds
print(min_unc)
print(best_bound)
submission_df["pred_95_low"] = np.maximum(preds_test - best_bound * preds_test_residual, 0)

submission_df["pred_95_high"] = np.maximum(preds_test + best_bound * preds_test_residual,0)

submission_df["prediction"] = np.maximum(preds_test, 0)
submission_df.to_csv("submissions/baseline.csv", index=False)
submission_df[submission_df.pred_95_high<submission_df.prediction]
submission_df.iloc[1272,submission_df.columns.get_loc('pred_95_low')] = 158984
submission_df.iloc[1272,submission_df.columns.get_loc('pred_95_high')] = 163442
submission_df.iloc[1272]['pred_95_low']= 158984
submission_df.iloc[1272].pred_95_high = 163442
submission_df[submission_df.pred_95_low>submission_df.prediction]
submission_df.describe()<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.11782407407407407, 'fraction_numerical': 0.02060185185185185, 'mean_word_length': 6.972324723247232, 'pattern_counts': {'":': 0, '<': 19, '<?xml version=': 0, '>': 18, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '4', 'dir': 'jupyter-structured-clean-dedup', 'id': '390039', 'license': 'MIT', 'n_tokens_mistral': 1710, 'n_tokens_neox': 1620, 'n_words': 236, 'path': 'Base.ipynb', 'repo_name': 'cmougan/Novartis2020', 'size': '4320'}