<jupyter_start><jupyter_text>Preprocessing<jupyter_code>from functools import cmp_to_key
from src.utils import composite_rating_cmp
isin       = pd.read_csv(RAW/'Isin.csv', low_memory=False)
ratings = list(isin.CompositeRating.value_counts().index)
ratings = sorted(ratings, key=cmp_to_key(composite_rating_cmp), reverse=True)
rank = {k: i for i, k in enumerate(ratings)}
%%time
for w in data_2018:
    w['CompositeRating']  = w.CompositeRating.apply(lambda x: rank[x])
cat_cols = ['BuySell', 'Sector', 'Subsector', 'Region_x', 'Country', 
            'TickerIdx', 'Seniority', 'Currency', 'ActivityGroup', 
            'Region_y', 'Activity', 'RiskCaptain', 'Owner', 
            'IndustrySector', 'IndustrySubgroup', 'MarketIssue', 'CouponType']
id_cols = ['TradeDateKey', 'CustomerIdx', 'IsinIdx']
target_col = 'CustomerInterest'
pred_col = 'PredictionIdx'
%%time
from src.utils import apply_cats
for col in cat_cols:
    data_2018[-1][col] = data_2018[-1][col].astype('category').cat.as_ordered()
for w in data_2018[:-1]:
    apply_cats(w, data_2018[-1])

for w in data_2018:
    for col in cat_cols:
        w[col] = w[col].cat.codes<jupyter_output>CPU times: user 7.71 s, sys: 528 ms, total: 8.24 s
Wall time: 8.24 s<jupyter_text>Model<jupyter_code>from src.utils import run_model
from lightgbm import LGBMClassifier
metric_names = ['auc']
for i, w in enumerate(data_2018[1:]):
    train, val, test = data_2018[i], w, data_2018[-1]
    print(train['TradeDateKey'].unique(), 
          val['TradeDateKey'].unique(), 
          test['TradeDateKey'].unique())
%%time
results = None
output = []
for i, w in enumerate(data_2018[1:]): 
    train, val, test = pd.concat(data_2018[:i+1]), w, data_2018[-1]
    X_train, y_train = train.drop(id_cols + [target_col], axis=1), \
                       train[target_col]
    if pred_col in val.columns: # when test acts as validation
        X_val, y_val = None, None
    else:
        X_val, y_val = val.drop(id_cols + [target_col], axis=1), \
                       val[target_col]
    X_test = test.drop(id_cols + [target_col, pred_col], axis=1)
        
    y_test, _, results, model = run_model(
            LGBMClassifier(n_estimators=400),
            X_train, y_train, X_val, y_val, X_test,
            metric_names, results, 
            params_desc='n_estimators=120',
            dataset_desc=f'{data_2018[i]}_diffcounts', 
            early_stopping_rounds=30)
    output.append([y_test, model])
rounds = [67, 364, 319, 109, 92, 82, 92, 225, 90, 233, 210, 301, 58, 26, 143, 120]
np.mean(rounds), np.median(rounds)
results['rounds'] = rounds
results
test[target_col] = output[-1][0]<jupyter_output><empty_output><jupyter_text>Submission<jupyter_code>submission = pd.read_csv(RAW/'sample_submission.csv', low_memory=False)
submission = pd.merge(submission[['PredictionIdx']], test[['PredictionIdx', target_col]], 
                      how='left', on='PredictionIdx')
submission[target_col].describe()
submission.head()
submission.to_csv(SUBMISSIONS/'19-lgbm_2018_diffscounts_0101-0416.csv', index=False)<jupyter_output><empty_output><jupyter_text>Feature importance<jupyter_code>from lightgbm import plot_importance
plot_importance(output[-1][1], figsize=(5,10))<jupyter_output><empty_output><jupyter_text>Train all data<jupyter_code>data_2018[-1].head()
all_2018 = pd.concat(data_2018[:-1])
all_2018.TradeDateKey.unique()
all_2018.shape
%%time
train = all_2018
test = data_2018[-1]
X_train, y_train = train.drop(id_cols + [target_col], axis=1), train[target_col]
X_test = test.drop(id_cols + [target_col, pred_col], axis=1)
assert list(X_train.columns) == list(X_test.columns)
y_test, _, results, model = run_model(
            LGBMClassifier(n_estimators=120),
            X_train, y_train, None, None, X_test,
            metric_names, results, 
            params_desc='n_estimators=120',
            dataset_desc=f'all_2018_diffcounts', 
            early_stopping_rounds=None)
results
test[target_col] = y_test
(test[target_col] - submission[target_col]).describe()<jupyter_output><empty_output><jupyter_text>Submission<jupyter_code>submission = pd.read_csv(RAW/'sample_submission.csv', low_memory=False)
submission = pd.merge(submission[['PredictionIdx']], test[['PredictionIdx', target_col]], 
                      how='left', on='PredictionIdx')
submission[target_col].describe()
submission.head()
submission.to_csv(SUBMISSIONS/'19-FIXED-lgbm_2018_diffscounts_0101-0416.csv', index=False)
from lightgbm import plot_importance
plot_importance(model, figsize=(5,10))<jupyter_output><empty_output><jupyter_text>Validate on previous month<jupyter_code>all_2018 = pd.concat(data_2018[:-2])
%%time
train = all_2018
test = data_2018[-2]
X_train, y_train = train.drop(id_cols + [target_col], axis=1), train[target_col]
X_test = test.drop(id_cols + [target_col], axis=1)
assert list(X_train.columns) == list(X_test.columns)
y_test, _, results, model = run_model(
            LGBMClassifier(n_estimators=120),
            X_train, y_train, None, None, X_test,
            metric_names, results, 
            params_desc='n_estimators=120',
            dataset_desc=f'all_2018_diffcounts', 
            early_stopping_rounds=None)
from sklearn.metrics import roc_auc_score
roc_auc_score(test[target_col], y_test)
challenge.shape, test.shape
from lightgbm import plot_importance
plot_importance(model, figsize=(5,10))
n_weeks = 5
jump = 1.25
dif = 0.5
acum = 1
mul = 0
weights = []
for ix in range(len(output)):
    if ix % n_weeks == 0:
        acum += dif
        mul = mul + acum
    weights.append(mul)
test = data_2018[-1]
test[target_col] = np.sum([x[0] * w for x, w in zip(output, weights)], axis=0) / sum(weights)<jupyter_output><empty_output><jupyter_text>Submission<jupyter_code>submission = pd.read_csv(RAW/'sample_submission.csv', low_memory=False)
submission = pd.merge(submission[['PredictionIdx']], test[['PredictionIdx', target_col]], 
                      how='left', on='PredictionIdx')
submission[target_col].describe()
submission.head()
submission.to_csv(SUBMISSIONS/'19-individual-lgbm_2018_diffscounts_0101-0416.csv', index=False)<jupyter_output><empty_output><jupyter_text>Comparison<jupyter_code>w04 = pd.read_feather(PROCESSED/f'all_weeks/week_{20180416}_diffscount.feather')
w04prev = pd.read_feather(PROCESSED/'week_0416_diffscount.feather')
w04.shape, w04prev.shape
w04.head()
w04prev = w04prev.iloc[:,:13]
w04prev.head()
w04.describe().transpose()
w04prev.describe().transpose()<jupyter_output><empty_output><jupyter_text>Fix Test<jupyter_code>from src.utils import get_weeks
week_labels = get_weeks(day_from=20160104, num_weeks=121)[104:]
week_labels
%%time
weeks = []
for name in week_labels[:-1]:
    weeks.append(pd.read_feather(PROCESSED/f'all_weeks/week_{name}_diffscount.feather'))
customer   = pd.read_csv(RAW/'Customer.csv', low_memory=False)
isin       = pd.read_csv(RAW/'Isin.csv', low_memory=False)
challenge  = pd.read_csv(RAW/'Challenge_20180423.csv', low_memory=False)
trade      = pd.read_csv(RAW/'Trade.csv', low_memory=False)
test = challenge.copy()
test['TradeDateKey'] = test['DateKey']
test.drop('DateKey', inplace=True, axis=1)
%%time
from src.utils import add_datediffs, add_dayscount
add_datediffs(test, trade, nan_date=20150701)
add_dayscount(test, trade)
test.to_feather(PROCESSED/f'all_weeks/week_20180423_diffscount.feather')
weeks.append(test)
%%time
for w in weeks:
    w['Month'] = w.TradeDateKey.apply(lambda x: x // 10**4 % 10**2)
    w['Day'] = w.TradeDateKey.apply(lambda x: x % 10**2)
%%time
from src.utils import preprocessing_pipeline
for i, w in enumerate(weeks):
    weeks[i] = preprocessing_pipeline(w, customer, isin, trade)<jupyter_output>CPU times: user 4.42 s, sys: 1.45 s, total: 5.88 s
Wall time: 5.87 s<jupyter_text>Preprocessing<jupyter_code>from functools import cmp_to_key
from src.utils import composite_rating_cmp
ratings = list(isin.CompositeRating.value_counts().index)
ratings = sorted(ratings, key=cmp_to_key(composite_rating_cmp), reverse=True)
rank = {k: i for i, k in enumerate(ratings)}
%%time
for w in weeks:
    w['CompositeRating']  = w.CompositeRating.apply(lambda x: rank[x])
cat_cols = ['BuySell', 'Sector', 'Subsector', 'Region_x', 'Country', 
            'TickerIdx', 'Seniority', 'Currency', 'ActivityGroup', 
            'Region_y', 'Activity', 'RiskCaptain', 'Owner', 
            'IndustrySector', 'IndustrySubgroup', 'MarketIssue', 'CouponType']
id_cols = ['TradeDateKey', 'CustomerIdx', 'IsinIdx']
target_col = 'CustomerInterest'
pred_col = 'PredictionIdx'
%%time
from src.utils import apply_cats
for col in cat_cols:
    weeks[-1][col] = weeks[-1][col].astype('category').cat.as_ordered()
for w in weeks[:-1]:
    apply_cats(w, weeks[-1])

for w in weeks:
    for col in cat_cols:
        w[col] = w[col].cat.codes<jupyter_output>CPU times: user 6.07 s, sys: 224 ms, total: 6.3 s
Wall time: 6.3 s<jupyter_text>Model<jupyter_code>from src.utils import run_model
from lightgbm import LGBMClassifier
metric_names = ['auc']
all_2018 = pd.concat(weeks[:-1])
all_2018.TradeDateKey.unique()
%%time
train = all_2018
test = weeks[-1]
X_train, y_train = train.drop(id_cols + [target_col], axis=1), train[target_col]
X_test = test.drop(id_cols + [target_col, pred_col], axis=1)
assert list(X_train.columns) == list(X_test.columns)
y_test, _, results, model = run_model(
            LGBMClassifier(n_estimators=120),
            X_train, y_train, None, None, X_test,
            metric_names, None, 
            params_desc='n_estimators=120',
            dataset_desc=f'all_2018_diffcounts', 
            early_stopping_rounds=None)
results
test[target_col] = y_test<jupyter_output><empty_output><jupyter_text>Submission<jupyter_code>submission = pd.read_csv(RAW/'sample_submission.csv', low_memory=False)
submission = pd.merge(submission[['PredictionIdx']], test[['PredictionIdx', target_col]], 
                      how='left', on='PredictionIdx')
submission[target_col].describe()
submission.head()
submission.to_csv(SUBMISSIONS/'19-TEST_FIXED-lgbm_2018_diffscounts_0101-0416.csv', index=False)<jupyter_output><empty_output><jupyter_text>Feature importance<jupyter_code>from lightgbm import plot_importance
plot_importance(model, figsize=(5,10))<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.14013299432818307, 'fraction_numerical': 0.041169567768433404, 'mean_word_length': 5.065836298932385, 'pattern_counts': {'":': 0, '<': 54, '<?xml version=': 0, '>': 54, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '14', 'dir': 'jupyter-structured-clean-dedup', 'id': '373981', 'license': 'MIT', 'n_tokens_mistral': 4067, 'n_tokens_neox': 3679, 'n_words': 660, 'path': 'notebooks/robert/RA-19-All-Weeks.ipynb', 'repo_name': 'Robert-Alonso/DSG-2018', 'size': '10226'}