<jupyter_start><jupyter_text><jupyter_code>import pandas as pd
import numpy as np
df_train = pd.read_csv('train.csv')
df_test = pd.read_csv('test.csv')
df_train.head()
print(df_train.info())
df_test.head()<jupyter_output><empty_output><jupyter_text>Data Cleaning<jupyter_code># droping id column
df_train.drop('id',axis=1,inplace = True)

# dropping duplicate values
df_train = df_train.drop_duplicates(keep='first')


# Null Values in train 
df_train.isna().sum()
# Null Values in test
df_test.isna().sum()
# dropping NA values in train
df_train = df_train.dropna()<jupyter_output><empty_output><jupyter_text>Vehicle Age<jupyter_code>df_train.Vehicle_Age.value_counts()
pip install category_encoders
import category_encoders as ce
df_train.Vehicle_Age
# create object of Ordinalencoding
encoder= ce.OrdinalEncoder(cols=['Vehicle_Age'],return_df=True,
                           mapping=[{'col':'Vehicle_Age',
                          'mapping':{'1-2 Year':1,'> 2 Years':2,'< 1 Year':0}}])

df_train_transformed = encoder.fit_transform(df_train)
df_train_transformed<jupyter_output><empty_output><jupyter_text>Region_code and Policy_sales_channel<jupyter_code>df_train_transformed.Region_Code.value_counts()
df_train_transformed.Policy_Sales_Channel.value_counts()[df_train_transformed.Policy_Sales_Channel.value_counts() < 5]
df_train_transformed.Policy_Sales_Channel.nunique()
# converting 'Policy_Sales_Channel','Region_Code' into categorical data type

df_train_transformed[['Policy_Sales_Channel','Region_Code']] = df_train_transformed[['Policy_Sales_Channel','Region_Code']].astype('category')
df_test[['Policy_Sales_Channel','Region_Code']] = df_test[['Policy_Sales_Channel','Region_Code']].astype('category')
#Common Policy_Sales_Channel in train and test data

streets = list(np.intersect1d(df_train_transformed.Policy_Sales_Channel, df_test['Policy_Sales_Channel']))
print('No. of Common Streets in train and test data are :\n', len(streets))

# **Replacing all the Policy_Sales_Channel by Others if not present in both train and test data**

df_train_transformed['Policy_Sales_Channel'] = ['Others' if x not in streets else x for x in df_train_transformed['Policy_Sales_Channel']  ]
df_test['Policy_Sales_Channel'] = ['Others' if x not in streets else x for x in df_test['Policy_Sales_Channel']  ]
#Common Region_Code in train and test data

streets = list(np.intersect1d(df_train_transformed.Region_Code, df_test['Region_Code']))
print('No. of Common Streets in train and test data are :\n', len(streets))

# **Replacing all the Region_Code by Others if not present in both train and test data**

df_train_transformed['Region_Code'] = ['Others' if x not in streets else x for x in df_train_transformed['Region_Code']  ]
df_test['Region_Code'] = ['Others' if x not in streets else x for x in df_test['Region_Code']  ]
df_train_transformed
df_test_transformed = encoder.fit_transform(df_test)
df_test_transformed
df_train_transformed.Region_Code = df_train_transformed.Region_Code.astype('category')
df_test_transformed.Region_Code = df_test_transformed.Region_Code.astype('category')<jupyter_output><empty_output><jupyter_text>Get Dummies<jupyter_code>df_train_dum = pd.get_dummies(df_train_transformed,drop_first=True)
df_test_dum = pd.get_dummies(df_test_transformed,drop_first=True)
df_test_dum.drop('id',axis=1,inplace=True)<jupyter_output><empty_output><jupyter_text>Normalise Data<jupyter_code>from sklearn.preprocessing import MinMaxScaler
scaler = MinMaxScaler()
df_test_dum.columns = df_test_dum.columns.str.replace('.','_')
df_train_dum.columns = df_train_dum.columns.str.replace('.','_')
[i for i in pd.get_dummies(df_test_dum.columns) if i not in df_train_dum.columns]
df = df_train_dum.drop('Response',axis=1)
y1 = df_train_dum.Response
scaled = scaler.fit(df)
df= pd.DataFrame(scaled.transform(df),columns=df.columns)
df_test_dum = pd.DataFrame(scaled.transform(df_test_dum),columns=df_test_dum.columns)<jupyter_output><empty_output><jupyter_text>feature importance<jupyter_code>features = df
labels = y1
from sklearn.ensemble import RandomForestClassifier
clf = RandomForestClassifier()

clf.fit(features,labels)

preds = clf.predict(features)
from pandas import DataFrame
VI = DataFrame(clf.feature_importances_, columns = ["RF"], index=features.columns)
VI = VI.reset_index()
VI.sort_values(['RF'],ascending=0)
VI['index']
imp = list(VI.sort_values(['RF'],ascending=0)['index'][0:100])
VII = VI.sort_values(['RF'],ascending=0)[0:100]
import pandas.core.algorithms as algos
from pandas import Series
import scipy.stats.stats as stats
import re
import traceback
import string

max_bin = 20
force_bin = 3

# define a binning function
def mono_bin(Y, X, n = max_bin):
    
    df1 = pd.DataFrame({"X": X, "Y": Y})
    justmiss = df1[['X','Y']][df1.X.isnull()]
    notmiss = df1[['X','Y']][df1.X.notnull()]
    r = 0
    while np.abs(r) < 1:
        try:
            d1 = pd.DataFrame({"X": notmiss.X, "Y": notmiss.Y, "Bucket": pd.qcut(notmiss.X, n)})
            d2 = d1.groupby('Bucket', as_index=True)
            r, p = stats.spearmanr(d2.mean().X, d2.mean().Y)
            n = n - 1 
        except Exception as e:
            n = n - 1

    if len(d2) == 1:
        n = force_bin         
        bins = algos.quantile(notmiss.X, np.linspace(0, 1, n))
        if len(np.unique(bins)) == 2:
            bins = np.insert(bins, 0, 1)
            bins[1] = bins[1]-(bins[1]/2)
        d1 = pd.DataFrame({"X": notmiss.X, "Y": notmiss.Y, "Bucket": pd.cut(notmiss.X, np.unique(bins),include_lowest=True)}) 
        d2 = d1.groupby('Bucket', as_index=True)
    
    d3 = pd.DataFrame({},index=[])
    d3["MIN_VALUE"] = d2.min().X
    d3["MAX_VALUE"] = d2.max().X
    d3["COUNT"] = d2.count().Y
    d3["EVENT"] = d2.sum().Y
    d3["NONEVENT"] = d2.count().Y - d2.sum().Y
    d3=d3.reset_index(drop=True)
    
    if len(justmiss.index) > 0:
        d4 = pd.DataFrame({'MIN_VALUE':np.nan},index=[0])
        d4["MAX_VALUE"] = np.nan
        d4["COUNT"] = justmiss.count().Y
        d4["EVENT"] = justmiss.sum().Y
        d4["NONEVENT"] = justmiss.count().Y - justmiss.sum().Y
        d3 = d3.append(d4,ignore_index=True)
    
    d3["EVENT_RATE"] = d3.EVENT/d3.COUNT
    d3["NON_EVENT_RATE"] = d3.NONEVENT/d3.COUNT
    d3["DIST_EVENT"] = d3.EVENT/d3.sum().EVENT
    d3["DIST_NON_EVENT"] = d3.NONEVENT/d3.sum().NONEVENT
    d3["WOE"] = np.log(d3.DIST_EVENT/d3.DIST_NON_EVENT)
    d3["IV"] = (d3.DIST_EVENT-d3.DIST_NON_EVENT)*np.log(d3.DIST_EVENT/d3.DIST_NON_EVENT)
    d3["VAR_NAME"] = "VAR"
    d3 = d3[['VAR_NAME','MIN_VALUE', 'MAX_VALUE', 'COUNT', 'EVENT', 'EVENT_RATE', 'NONEVENT', 'NON_EVENT_RATE', 'DIST_EVENT','DIST_NON_EVENT','WOE', 'IV']]       
    d3 = d3.replace([np.inf, -np.inf], 0)
    d3.IV = d3.IV.sum()
    
    return(d3)

def char_bin(Y, X):
        
    df1 = pd.DataFrame({"X": X, "Y": Y})
    justmiss = df1[['X','Y']][df1.X.isnull()]
    notmiss = df1[['X','Y']][df1.X.notnull()]    
    df2 = notmiss.groupby('X',as_index=True)
    
    d3 = pd.DataFrame({},index=[])
    d3["COUNT"] = df2.count().Y
    d3["MIN_VALUE"] = df2.sum().Y.index
    d3["MAX_VALUE"] = d3["MIN_VALUE"]
    d3["EVENT"] = df2.sum().Y
    d3["NONEVENT"] = df2.count().Y - df2.sum().Y
    
    if len(justmiss.index) > 0:
        d4 = pd.DataFrame({'MIN_VALUE':np.nan},index=[0])
        d4["MAX_VALUE"] = np.nan
        d4["COUNT"] = justmiss.count().Y
        d4["EVENT"] = justmiss.sum().Y
        d4["NONEVENT"] = justmiss.count().Y - justmiss.sum().Y
        d3 = d3.append(d4,ignore_index=True)
    
    d3["EVENT_RATE"] = d3.EVENT/d3.COUNT
    d3["NON_EVENT_RATE"] = d3.NONEVENT/d3.COUNT
    d3["DIST_EVENT"] = d3.EVENT/d3.sum().EVENT
    d3["DIST_NON_EVENT"] = d3.NONEVENT/d3.sum().NONEVENT
    d3["WOE"] = np.log(d3.DIST_EVENT/d3.DIST_NON_EVENT)
    d3["IV"] = (d3.DIST_EVENT-d3.DIST_NON_EVENT)*np.log(d3.DIST_EVENT/d3.DIST_NON_EVENT)
    d3["VAR_NAME"] = "VAR"
    d3 = d3[['VAR_NAME','MIN_VALUE', 'MAX_VALUE', 'COUNT', 'EVENT', 'EVENT_RATE', 'NONEVENT', 'NON_EVENT_RATE', 'DIST_EVENT','DIST_NON_EVENT','WOE', 'IV']]      
    d3 = d3.replace([np.inf, -np.inf], 0)
    d3.IV = d3.IV.sum()
    d3 = d3.reset_index(drop=True)
    
    return(d3)

def data_vars(df1, target):
    
    stack = traceback.extract_stack()
    filename, lineno, function_name, code = stack[-2]
    vars_name = re.compile(r'\((.*?)\).*$').search(code).groups()[0]
    final = (re.findall(r"[\w']+", vars_name))[-1]
    
    x = df1.dtypes.index
    count = -1
    
    for i in x:
        if i.upper() not in (final.upper()):
            if np.issubdtype(df1[i], np.number) and len(Series.unique(df1[i])) > 2:
                conv = mono_bin(target, df1[i])
                conv["VAR_NAME"] = i
                count = count + 1
            else:
                conv = char_bin(target, df1[i])
                conv["VAR_NAME"] = i            
                count = count + 1
                
            if count == 0:
                iv_df = conv
            else:
                iv_df = iv_df.append(conv,ignore_index=True)
    
    iv = pd.DataFrame({'IV':iv_df.groupby('VAR_NAME').IV.max()})
    iv = iv.reset_index()
    return(iv_df,iv)
final_iv, IV = data_vars(features[imp],labels)
final_iv
IV = IV.rename(columns={'VAR_NAME':'index'})
IV.sort_values(['IV'],ascending=0)
IV.sort_values(['IV'],ascending=0)[0:20]
IV[IV['IV'] != 0.000000].sort_values(['IV'],ascending=0)
from sklearn.feature_selection import RFE
from sklearn.linear_model import LogisticRegression

model = LogisticRegression()
rfe = RFE(model, 20)
fit = rfe.fit(features[imp], labels)
from pandas import DataFrame
Selected = DataFrame(rfe.support_, columns = ["RFE"], index=features[imp].columns)
Selected = Selected.reset_index()
Selected[Selected['RFE'] == True]
from sklearn.ensemble import ExtraTreesClassifier

model = ExtraTreesClassifier()
model.fit(features[imp], labels)

print(model.feature_importances_)
from pandas import DataFrame
FI = DataFrame(model.feature_importances_, columns = ["Extratrees"], index=features[imp].columns)
FI = FI.reset_index()
FI.sort_values(['Extratrees'],ascending=0)
from sklearn.feature_selection import SelectKBest
from sklearn.feature_selection import chi2

model = SelectKBest(score_func=chi2, k=5)
fit = model.fit(features[imp].abs(), labels)
from pandas import DataFrame
pd.options.display.float_format = '{:.2f}'.format
chi_sq = DataFrame(fit.scores_, columns = ["Chi_Square"], index=features[imp].columns)
chi_sq = chi_sq.reset_index()
chi_sq.sort_values('Chi_Square',ascending=0)
from sklearn.svm import LinearSVC
from sklearn.feature_selection import SelectFromModel
lsvc = LinearSVC(C=0.01, penalty="l1", dual=False).fit(features[imp], labels)
model = SelectFromModel(lsvc,prefit=True)
from pandas import DataFrame
l1 = DataFrame(model.get_support(), columns = ["L1"], index=features[imp].columns)
l1 = l1.reset_index()
from functools import reduce
dfs = [IV, VII, Selected, FI, chi_sq, l1]
final_results = reduce(lambda left,right: pd.merge(left,right,on='index'), dfs)
columns = ['IV', 'RF', 'Extratrees', 'Chi_Square']

score_table = pd.DataFrame({},[])
score_table['index'] = final_results['index']

for i in columns:
    score_table[i] = final_results['index'].isin(list(final_results.nlargest(5,i)['index'])).astype(int)
    
score_table['RFE'] = final_results['RFE'].astype(int)
score_table['L1'] = final_results['L1'].astype(int)
score_table['final_score'] = score_table.sum(axis=1)
score_table.sort_values('final_score',ascending=0)
features = features[list(score_table[score_table['final_score'] >= 2]['index'])]<jupyter_output><empty_output><jupyter_text>MultiCollinearity<jupyter_code>from statsmodels.stats.outliers_influence import variance_inflation_factor
def calculate_vif(features):
    vif = pd.DataFrame()
    vif["Features"] = features.columns
    vif["VIF"] = [variance_inflation_factor(features.values, i) for i in range(features.shape[1])]    
    return(vif)
vif = calculate_vif(features)
while vif['VIF'][vif['VIF'] > 10].any():
    remove = vif.sort_values('VIF',ascending=0)['Features'][:1]
    features.drop(remove,axis=1,inplace=True)
    vif = calculate_vif(features)
list(vif['Features'])
final = df[list(vif['Features'])]<jupyter_output><empty_output><jupyter_text>Smote<jupyter_code>from imblearn.over_sampling import SMOTE
oversample = SMOTE()
X, y = oversample.fit_resample(df, y1)<jupyter_output><empty_output><jupyter_text>catboost<jupyter_code>from catboost import CatBoostClassifier

from sklearn.model_selection import StratifiedKFold
from sklearn.model_selection import GridSearchCV
from sklearn.model_selection import train_test_split
# Model selection
from sklearn.model_selection import StratifiedKFold
# Metrics
from sklearn.metrics import roc_auc_score
from sklearn.metrics import make_scorer
# Skopt functions
from skopt import BayesSearchCV
from skopt.callbacks import DeadlineStopper, VerboseCallback, DeltaXStopper
from skopt.space import Real, Categorical, Integer
from time import time
X_train, X_valid, y_train, y_valid = train_test_split(X,y, 
                                                      test_size=0.3, 
                                                      random_state=17)


base_model = CatBoostClassifier(iterations=1000,
    learning_rate=0.1,
    random_strength=0.1,
    depth=8,
    loss_function='',
    eval_metric='Accuracy',
    leaf_estimation_method='Newton')
base_model.fit(X,y)


y_pred = [i[0] for i in base_model.predict(df_test_dum)]
y_pred
s1 = pd.concat([df_test.id,pd.Series(y_pred)],axis=1)
s1.columns = ['id','Response']
s1.to_csv('s1.csv',index=False)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.15153297551650613, 'fraction_numerical': 0.018013381369016986, 'mean_word_length': 4.5495716034271725, 'pattern_counts': {'":': 10, '<': 44, '<?xml version=': 0, '>': 47, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '10', 'dir': 'jupyter-structured-clean-dedup', 'id': '601511', 'license': 'MIT', 'n_tokens_mistral': 5137, 'n_tokens_neox': 4946, 'n_words': 858, 'path': 'cross_sell_AV.ipynb', 'repo_name': 'saurabhec1091/ML-Micro-Projects-and-Tutorials', 'size': '13601'}