<jupyter_start><jupyter_text>Parameter tunings  Test dataset predictions<jupyter_code>### Prediction for test datasets by different XGB methods
## combined_avg_pred_xgb_lgb   ## From ensemble section

### Create renewal predictions df
subm_df = pd.DataFrame()
subm_df['id'] = test_df['id']
subm_df['renewal'] = combined_avg_pred_xgb_lgb
subm_df['incentives_actual'] = 0

### Create incentive predictions df
prem_limit_incentive = 100
subm_df['delta_perc'] = (1 - subm_df['renewal'])
subm_df['incentives_actual'] = subm_df['delta_perc'].apply(get_incentives)
pred_test_df = pd.merge(subm_df, test_df, on='id', how='right', suffixes=('_pred', '_orig'))
pred_test_df['incentives_limited'] = pred_test_df[['premium','incentives_actual']].apply(lambda x: func1(x, prem_limit_incentive), axis=1)

pred_test_df.loc[pred_test_df['incentives_limited'] != pred_test_df['incentives_actual'], :]
subm_df['incentives'] = pred_test_df['incentives_limited'] + 350
# subm_df['incentives'] = pred_test_df['incentives_actual']
subm_df.head()
subm_df.info()

# subm_df[['id', 'renewal', 'incentives']].to_csv(data_path + 'day3_3_ens_xgb_lgb.csv', index=False) ## 0.7207
# subm_df[['id', 'renewal', 'incentives']].to_csv(data_path + 'day3_4_ens_xgb_lgb.csv', index=False) ## 0.7207
# subm_df[['id', 'renewal', 'incentives']].to_csv(data_path + 'subm_day3_6_ens_xgb_lgb.csv', index=False) ## 0.7251
# subm_df[['id', 'renewal', 'incentives']].to_csv(data_path + 'subm_day3_7_ens_xgb_lgb.csv', index=False) ## 0.723
subm_df[['id', 'renewal', 'incentives']].to_csv(data_path + 'subm_day3_9_ens_xgb_lgb.csv', index=False) ## 0.7248


sum(test_df['premium'])
max_prop
# pred_test_df['delta_perc'].apply(lambda x: max_prop if x > max_prop else x)<jupyter_output><empty_output><jupyter_text>Rough work<jupyter_code>pd.cut(subm_df['renewal'],10).value_counts().sort_index()

# pd.cut(subm_df['renewal'],10).sort_index()

import math
from collections import defaultdict
incentive_amt = 1200

incentive_range=range(50,1300,50)
incentive_dict = defaultdict(np.float)
calc_incentives = []
calc_effort = []
calc_prob = []


for amt in incentive_range:
    effort_incentive = 10*(1-math.exp(-amt/400))
    prob_effort = 20*(1-math.exp(-effort_incentive/5))
    calc_incentives.append(amt)
    calc_effort.append(effort_incentive)
    calc_prob.append(prob_effort)

calc_incentives
calc_effort
calc_prob

len_calc_prob = len(calc_prob)

calc_prob_tuples = []
for i in range(0, len_calc_prob-1):
    calc_prob_tuples.append((calc_prob[i], calc_prob[i+1]))

calc_prob_tuples<jupyter_output><empty_output><jupyter_text>ensemble LGB<jupyter_code>## Base LGB run
lgb_train = lgb.Dataset(X_train, y_train)
lgb_eval = lgb.Dataset(X_val, y_val, reference=lgb_train)

lgb_params1 = {
    'task': 'train',
    'boosting_type': 'gbdt',
    'objective': 'binary',
    'metric': 'auc',
    'num_leaves': 70,
    'learning_rate': 0.02,
    'feature_fraction': 0.8,
#     'colsample_by_tree':0.8,
    'bagging_fraction': 0.8,
    'bagging_freq': 3,
#     'verbose': 100, 
   'min_data_in_leaf': 10, 
   'max_depth' : 5,
}


ens_subm_lgb1 = pd.DataFrame()
for idx, curr_seed in enumerate(range(1000,100000,1357)):
    print('[{0}] Processing for seed: {1}'.format(idx, curr_seed))
    col_name = 'lgb_{0}'.format(idx)
    if idx == 10:
        break
        
    lgb_params1['feature_fraction_seed'] =  curr_seed
#     lgb_params1['bagging_fraction_seed'] = round(np.random.rand()*curr_seed)
    lgb_params1['lambda_l1'] = 10 ** (np.random.rand() * 4)
    lgb_params1['lambda_l2'] = 10 ** (np.random.rand() * 3 + 2)   
        
    lgb1 = lgb.train(lgb_params1, lgb_train, num_boost_round=5000, valid_sets=lgb_eval, early_stopping_rounds=200, verbose_eval=500)
    ens_subm_lgb1[col_name] = lgb1.predict(X_test)
    
ens_subm_lgb1.head(20)
def combine_avg_models(df):
    return np.mean(df, axis=1)
# del ens_subm_lgb1['lgb_8']
ens_subm_lgb1.head()
### Combine XGB and LGB ensembles
# xgb1_ens_df = pd.read_csv(data_path + 'day3_2_fullpred.csv')
xgb1_ens_df = pd.read_csv(data_path + 'day3_6_xgb_fullpred.csv')

xgb1_ens_df.info()
xgb1_ens_df.head()

all_pred_xgb1_lgb1 = pd.concat([xgb1_ens_df,ens_subm_lgb1], axis=1 )
all_pred_xgb1_lgb1.head()


combined_avg_pred_xgb_lgb = combine_avg_models(all_pred_xgb1_lgb1)
combined_avg_pred_xgb_lgb
# all_pred_xgb1_lgb1.to_csv(data_path + 'day3_3_xgb_lgb_fullpred.csv', index=False)
# all_pred_xgb1_lgb1.to_csv(data_path + 'day3_6_xgb_lgb_fullpred.csv', index=False)
### Incentive analysis
test_df.head()
temp_df = pd.merge(subm_df[['id', 'renewal', 'incentives', 'delta_perc']], test_df[['id', 'premium', 'premium_bands', 'policy_late_pcnt', 'appl_underwrite_score']], how='inner', on='id',
        suffixes = ('_x', '_y'))
temp_df.info()

temp_df['incentives_250'] = temp_df['incentives'] + 250
temp_df['incentives_300'] = temp_df['incentives'] + 300
temp_df['incentives_500'] = temp_df['incentives'] + 500
temp_df['appl_underwrite_score_needed'] = temp_df['appl_underwrite_score'].apply(lambda x: 4 if x < 0 else round(100-x))

temp_df['delta_perc_round'] = round(temp_df['delta_perc']*100)
temp_df.head()
temp_df['appl_underwrite_score_needed'].value_counts()
# total_net_revenue_724 = sum(temp_df['incentives_250'])
# total_net_revenue_725 = sum(temp_df['incentives_300'])
# total_net_revenue_723 = sum(temp_df['incentives_500'])


total_net_revenue_724
total_net_revenue_725
total_net_revenue_723

# temp_df.groupby(['premium_bands', 'delta_perc_round']).agg({'incentives': 'mean', 'incentives_300': 'mean'})

premium_weight = range(80,220,3)
delta_weight = range(80,225,3)

def func_apply_weight(row, pw, dw):
    return ((row['premium_bands']+1) * pw) + (row['appl_underwrite_score_needed'] * dw ) + row['incentives']

# for pw in premium_weight:
#     for dw in delta_weight:
#         temp_df['new_incentive'] = temp_df.apply(lambda x: func_apply_weight(x, pw, dw), axis=1)
#         curr_tot_revenue = sum(temp_df['new_incentive'])
# #         print('Total net revenue: {0}'.format(curr_tot_revenue))
#         print('Pending net revenue: {0} % [{1}, {2}]'.format( round((total_net_revenue_725 - curr_tot_revenue)*100/total_net_revenue_725,2), pw, dw))

#     #### This looks interesting --> Pending net revenue: -0.14 % [80, 119]

temp_df['new_incentive'] = temp_df.apply(lambda x: func_apply_weight(x, 80, 119), axis=1)

sum(temp_df['new_incentive'])

temp_df['incentives_350'] = temp_df['incentives'] + 350
sum(temp_df['incentives_350'])

#### Write adjusted incentives for same renewal and same 300
temp2_df = temp_df[['id', 'renewal', 'new_incentive']]
temp2_df.columns = ['id', 'renewal', 'incentives']
temp2_df.to_csv(data_path + 'subm_day3_8_ens_xgb_lgb.csv', index=False) ##<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.15787905868334823, 'fraction_numerical': 0.042597557342865654, 'mean_word_length': 6.709529276693456, 'pattern_counts': {'":': 0, '<': 14, '<?xml version=': 0, '>': 15, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '3', 'dir': 'jupyter-structured-clean-dedup', 'id': '543531', 'license': 'MIT', 'n_tokens_mistral': 2968, 'n_tokens_neox': 2791, 'n_words': 398, 'path': 'Mckinsey_Ins_Boost_LGB.ipynb', 'repo_name': 'shankarrajus/Hack_Mckinsey_Insurance', 'size': '6714'}