<jupyter_start><jupyter_text>Important custom functions<jupyter_code>def savePredictions(model,fileName,test_values):
    y_pred = model.predict_proba(test_values)[:,1]
    sub['Response'] = y_pred
    sub.to_csv(fileName+'.csv',index=False)
    return sub
scores = [0]

def getInfo(dataset):
    info = pd.DataFrame({'Nunuique':dataset.nunique(),'DataType':dataset.dtypes,'NullValues':dataset.isnull().sum()})
    return info

def preprocess(train):
    sc = StandardScaler()
    train = sc.fit_transform(train)
    return train<jupyter_output><empty_output><jupyter_text>Append train and test<jupyter_code>#train = train[train.Annual_Premium < 200000]
#train['Annual_Premium']=np.log(train['Annual_Premium'])
#train = train.sample(frac=1).reset_index(drop=True)
raw_data = train.append(test)
raw_data.index = range(len(raw_data))
dataset = raw_data.copy()<jupyter_output><empty_output><jupyter_text>Mapping<jupyter_code>vehicle_age_map = {"> 2 Years":3,"1-2 Year":2,"< 1 Year":1}
dataset["Vehicle_Age"] = dataset["Vehicle_Age"].map(vehicle_age_map)<jupyter_output><empty_output><jupyter_text>Adding new features and binning<jupyter_code># adding new features and binning

l = []
for i in raw_data['Age']:
    if(i<25):l.append(1)
    elif(i>=25 and i<35):l.append(3)
    elif(i>=35 and i<50):l.append(4)
    elif(i>=50 and i<60):l.append(5)
    else:l.append(2)
dataset['Salary'] = l

Bins_Age = np.linspace(dataset['Age'].min(),dataset['Age'].max(),6)
Bins_Annual_Premium = np.linspace(dataset['Annual_Premium'].min(),dataset['Annual_Premium'].max(),4)
Bins_Vintage = np.linspace(dataset['Vintage'].min(),dataset['Vintage'].max(),4)

dataset['Experience_level'] =  pd.cut(dataset['Age'] , Bins_Age , labels=['one','two','three','four','five'],include_lowest=True)
dataset['Category_of_Annual_Premium'] =  pd.cut(dataset['Annual_Premium'] , Bins_Annual_Premium , labels=['low','medium','high'],include_lowest=True)
dataset['Time_of_Trust'] = pd.cut(dataset['Vintage'] , Bins_Vintage , labels=['short','average','long'],include_lowest=True)

transform_series = dataset.groupby('Vehicle_Age').size()/len(dataset)
dataset['Vehicle_Age_ratio']= dataset['Vehicle_Age'].apply(lambda x: transform_series[x])

transform_series = dataset.groupby('Policy_Sales_Channel').size()/len(dataset)
dataset['Policy_Sales_Channel_ratio']= dataset['Policy_Sales_Channel'].apply(lambda x: transform_series[x])


dataset['Experience_level'] =  pd.cut(dataset['Age'] , Bins_Age , labels=['one','two','three','four','five'],include_lowest=True)
dataset['Category_of_Annual_Premium'] =  pd.cut(dataset['Annual_Premium'] , Bins_Annual_Premium , labels=['low','medium','high'],include_lowest=True)

dataset['Annual_Premium']=np.log(dataset['Annual_Premium'])
#db = dataset.groupby(['Gender','Vehicle_Damage']).size() 
#db2 = dataset.groupby(['Gender']).size()
#dataset['Probab_previously_damage_per_gender'] = dataset['Gender'].apply(lambda x:db[x][1]/db2[x])<jupyter_output><empty_output><jupyter_text>Label Encode the columns<jupyter_code>columns_to_be_label_encoded = ['Gender','Vehicle_Damage','Category_of_Annual_Premium','Experience_level','Time_of_Trust']
for col in columns_to_be_label_encoded:
    le = LabelEncoder()
    dataset[col] = le.fit_transform(dataset[col])<jupyter_output><empty_output><jupyter_text>Feature Engineering<jupyter_code>df = dataset.copy()
df['Damage_sums_per_region'] = df.groupby(['Region_Code'])['Vehicle_Damage'].transform('sum')
df['Mean_premium_per_region'] = df.groupby(['Region_Code'])['Annual_Premium'].transform('mean')
df['Count_unique_policy_sales_per_region'] = df.groupby(['Region_Code'])['Policy_Sales_Channel'].transform('nunique')
df['Count_policy_sales_per_region'] = df.groupby(['Region_Code'])['Policy_Sales_Channel'].transform('count')
df['Mean_vehicle_age_per_region'] = df.groupby(['Region_Code'])['Vehicle_Age'].transform('mean')
df['Mean_age_per_region'] = df.groupby(['Region_Code'])['Age'].transform('mean')
df['Mean_salary_per_region'] = df.groupby(['Region_Code'])['Salary'].transform('mean')
df['Count_previously_insured_per_region'] = df.groupby(['Region_Code'])['Previously_Insured'].transform('sum')
df['Mean_vintage_per_region'] = df.groupby(['Region_Code'])['Vintage'].transform('mean')
df['Max_premimum_per_region'] = df.groupby(['Region_Code'])['Annual_Premium'].transform('max')
df['Max_premimum_per_region'] = df.groupby(['Region_Code'])['Annual_Premium'].transform('min')
df["Rank_premium_per_region"] = df.groupby("Region_Code")['Annual_Premium'].rank(method="dense", ascending=True)

df['Mean_cat_premium_per_experience'] = df.groupby(["Experience_level"])['Category_of_Annual_Premium'].transform('mean')
df['Mean_premium_per_experience'] = df.groupby(["Experience_level"])['Annual_Premium'].transform('mean')
df['Mean_salary_per_experience'] = df.groupby(["Experience_level"])['Salary'].transform('mean')
#df['Mean_vehicle_damage_per_experience'] = df.groupby(["Experience_level"])['Vehicle_Damage'].transform('mean')
#df['Mean_vehicle_age_per_experience'] = df.groupby(["Experience_level"])['Vehicle_Age'].transform('mean')
#df['no_of_people_from_regionwise'] = df.groupby(['Region_Code'])['id'].transform('nunique')
#df['no_of_people_in_policy_channel'] = df.groupby(['Policy_Sales_Channel'])['id'].transform('count')
#df['no_of_people_with_same_associatedDays'] = df.groupby(['Vintage'])['id'].transform('count')
#df["RANK"] = df.groupby("Region_Code")['Region_Code'].rank(method="first", ascending=True)
#df["RANK_avg"] = df.groupby("Region_Code")['Region_Code'].rank(method="average", ascending=True)
#df["RANK_max"] = df.groupby("Region_Code")['Region_Code'].rank(method="max", ascending=True)
#df["RANK_min"] = df.groupby("Region_Code")['Region_Code'].rank(method="min", ascending=True)
#df["RANK_DIFF"] = df['RANK_max'] - df['RANK_min']

#df["RANK_Vehicle_Age"] = df.groupby(['Region_Code','Vehicle_Age'])['Region_Code'].rank(method='first',ascending=True)

policy_dict_map = df['Policy_Sales_Channel'].value_counts().to_dict()

#df['feaquency_count_policy']= df['Policy_Sales_Channel'].map(policy_dict_map)
from sklearn.feature_selection import VarianceThreshold
var_thresh = VarianceThreshold(threshold=0.1)
selector = var_thresh.fit(df)

df = df[df.columns[selector.get_support(indices=True)]]
df
#getInfo(df)<jupyter_output><empty_output><jupyter_text>Dividing into train and test<jupyter_code>train_df = df[df['Response'].isnull()==False]
test_df= df[df['Response'].isnull()==True]
X = train_df.drop(['Response','id'],axis=1)
y = train_df['Response'] 
X_pred =test_df.drop(['Response','id'],axis=1)
X.shape , y.shape , X_pred.shape<jupyter_output><empty_output><jupyter_text>train_test_split of the train and the test<jupyter_code>X_train,X_test,y_train,y_test = train_test_split(X,y,test_size=0.15,random_state =2020,stratify=y)
X_train.shape , X_test.shape , y_train.shape , y_test.shape<jupyter_output><empty_output><jupyter_text>Employing various Models for the classification Task  1) Lightgbm<jupyter_code>lgb = lg.LGBMClassifier(boosting_type='gbdt',
                     n_estimators=450,learning_rate=0.0295,
                     objective='binary',metric='auc',is_unbalance=True,
                     colsample_bytree=0.5,reg_lambda=10,
                     reg_alpha=2,random_state=294,n_jobs=-1,seed=8798,max_depth=7 , max_bin=None,
                        early_stopping=20
                    )
lgb.fit(X_train,y_train,eval_set=[(X_test,y_test),(X_train, y_train)])
lg.plot_metric(lgb)
score = roc_auc_score(y_test,lgb.predict_proba(X_test,lgb.best_iteration_)[:,1])
scores.append(score)
print(scores)
sub = savePredictions(lgb,'LightGBM',X_pred)
sub
l=[8907 , 567 , 3412 , 2030]
for i in l:
    lgb = lg.LGBMClassifier(boosting_type='gbdt',
                     n_estimators=450,depth=10,learning_rate=0.03,
                     objective='binary',metric='auc',is_unbalance=True,
                     colsample_bytree=0.5,reg_lambda=10,
                     reg_alpha=2,random_state=294,n_jobs=-1,seed=i,max_depth=7
                    )
    lgb.fit(X_train,y_train,eval_set=[(X_test,y_test),(X_train, y_train)],verbose=False)
    score = roc_auc_score(y_test,lgb.predict_proba(X_test)[:,1])
    print(score)
l=[None,20,60,100,200]
for i in l:
    lgb = lg.LGBMClassifier(boosting_type='gbdt',
                     n_estimators=450,depth=10,learning_rate=0.03,
                     objective='binary',metric='auc',is_unbalance=True,
                     colsample_bytree=0.5,reg_lambda=10,
                     reg_alpha=2,random_state=294,n_jobs=-1,seed=2030,max_depth=7,max_bin=i
                    )
    lgb.fit(X_train,y_train,eval_set=[(X_test,y_test),(X_train, y_train)],verbose=False)
    score = roc_auc_score(y_test,lgb.predict_proba(X_test)[:,1])
    print(score)
l=[-1,10,20,7,4]
for i in l:
    lgb = lg.LGBMClassifier(boosting_type='gbdt',
                     n_estimators=450,depth=10,learning_rate=0.03,
                     objective='binary',metric='auc',is_unbalance=True,
                     colsample_bytree=0.5,reg_lambda=10,
                     reg_alpha=2,random_state=294,n_jobs=-1,seed=2030,max_depth=i,max_bin=None
                    )
    lgb.fit(X_train,y_train,eval_set=[(X_test,y_test),(X_train, y_train)],verbose=False)
    score = roc_auc_score(y_test,lgb.predict_proba(X_test)[:,1])
    print(score)
from sklearn.feature_selection import chi2
from sklearn.feature_selection import f_classif
from sklearn.feature_selection import mutual_info_classif
from sklearn.feature_selection import SelectKBest
from sklearn.feature_selection import SelectPercentile

ufs = SelectKBest(chi2,k=25)
ufs.fit(X, y)
X_transformed = ufs.transform(X)
X_pred = ufs.transform(X_pred)


from sklearn.feature_selection import SelectFromModel
col_names = X.columns

lgb = lg.LGBMClassifier(boosting_type='gbdt',
                     n_estimators=450,learning_rate=0.0285,
                     objective='binary',metric='auc',is_unbalance=True,
                     colsample_bytree=0.5,reg_lambda=10,
                     reg_alpha=2,random_state=294,n_jobs=-1,seed=8798,max_depth=7 , max_bin=None)
sfm = SelectFromModel(estimator=lgb)
X_transformed = sfm.fit_transform(X, y)
# see which features were selected
support = sfm.get_support()
# get feature names
print([
 x for x, y in zip(col_names, support) if y == True
])
cols=['Age', 'Region_Code', 'Vehicle_Age', 'Annual_Premium', 'Policy_Sales_Channel', 
      'Vintage', 'Mean_premium_per_region', 'Mean_age_per_region', 'Mean_salary_per_region', 'Rank_premium_per_region']
train_df = df[df['Response'].isnull()==False]
test_df= df[df['Response'].isnull()==True]

X = train_df[cols]
y = train_df['Response'] 
X_pred =test_df[cols]

X.shape , X_pred.shape
lgb = lg.LGBMClassifier(boosting_type='gbdt',
                     n_estimators=450,learning_rate=0.0295,
                     objective='binary',metric='auc',is_unbalance=True,
                     colsample_bytree=0.5,reg_lambda=10,
                     reg_alpha=2,random_state=294,n_jobs=-1,seed=8798,max_depth=7 , max_bin=None)
lgb.fit(X,y)
py = lgb.predict_proba(X_pred)[:,1]
sub = savePredictions(lgb,'LightGBM',X_pred)
features = list(X.columns)

param = {
    'bagging_freq': 5,
    'bagging_fraction': 0.4,
    'boost_from_average':'false',
    'boost': 'gbdt',
    'feature_fraction': 0.05,
    'learning_rate': 0.01,
    'max_depth': -1,  
    'metric':'auc',
    'min_data_in_leaf': 80,
    'min_sum_hessian_in_leaf': 10.0,
    'num_leaves': 13,
    'num_threads': 8,
    'tree_learner': 'serial',
    'objective': 'binary', 
    'verbosity': 1
}

folds = StratifiedKFold(n_splits=5, shuffle=False, random_state=44000)
oof = np.zeros(len(train_df))
predictions = np.zeros(len(test_df))
feature_importance_df = pd.DataFrame()

for fold_, (trn_idx, val_idx) in enumerate(folds.split(X, y)):
    print("Fold {}".format(fold_))
    trn_data = lg.Dataset(X.iloc[trn_idx][features], label=y.iloc[trn_idx])
    val_data = lg.Dataset(X.iloc[val_idx][features], label=y.iloc[val_idx])

    num_round = 10000
    clf = lg.train(param, trn_data, num_round, valid_sets = [trn_data, val_data], verbose_eval=1000, early_stopping_rounds = 3000)
    oof[val_idx] = clf.predict(train_df.iloc[val_idx][features], num_iteration=clf.best_iteration)
    
    fold_importance_df = pd.DataFrame()
    fold_importance_df["Feature"] = features
    fold_importance_df["importance"] = clf.feature_importance()
    fold_importance_df["fold"] = fold_ + 1
    feature_importance_df = pd.concat([feature_importance_df, fold_importance_df], axis=0)
    
    predictions += clf.predict(X_pred, num_iteration=clf.best_iteration) / folds.n_splits

print("CV score: {:<8.5f}".format(roc_auc_score(target, oof)))
print("CV score: {:<8.5f}".format(roc_auc_score(target.iloc[val_idx], oof)))
sub["Response"] = predictions
sub.to_csv("last.csv", index=False)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1653708598973721, 'fraction_numerical': 0.020758824444098895, 'mean_word_length': 6.31683731513083, 'pattern_counts': {'":': 3, '<': 45, '<?xml version=': 0, '>': 41, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '9', 'dir': 'jupyter-structured-clean-dedup', 'id': '459672', 'license': 'MIT', 'n_tokens_mistral': 4973, 'n_tokens_neox': 4709, 'n_words': 514, 'path': 'Cross selling prediction/model analysis.ipynb', 'repo_name': 'harsh204016/Hackathon-and-Competitions', 'size': '12862'}