<jupyter_start><jupyter_text>Libraries used<jupyter_code>import pandas as pd
import numpy as np
from sklearn.linear_model import LogisticRegression
from sklearn.tree import DecisionTreeClassifier, ExtraTreeClassifier
from sklearn.svm import LinearSVC
from sklearn.ensemble import RandomForestClassifier, AdaBoostClassifier, GradientBoostingClassifier
from sklearn.pipeline import make_pipeline
from sklearn.preprocessing import StandardScaler
from sklearn.impute import SimpleImputer
from sklearn.model_selection import cross_val_score
import statsmodels.api as sm
from sklearn.compose import make_column_transformer, ColumnTransformer
from sklearn.feature_selection import SelectKBest, SelectPercentile, f_classif, \
mutual_info_classif, SelectFromModel, RFE
import warnings
warnings.filterwarnings('ignore')<jupyter_output><empty_output><jupyter_text>Read data<jupyter_code>train = pd.read_csv('train.csv')
test = pd.read_csv('test.csv')
X_train = train.iloc[:,2:]
y_train = train.iloc[:,1]
X_test = test.iloc[:,1:]<jupyter_output><empty_output><jupyter_text>Generate output csv for submission<jupyter_code>def genop(pred):
    df = pd.DataFrame()
    df['id'] = test['id']
    df['target'] = pred
    df.to_csv('output.csv', index=False)<jupyter_output><empty_output><jupyter_text>Check correlation of features<jupyter_code>def get_redundant_pairs(df):
    '''Get diagonal and lower triangular pairs of correlation matrix'''
    pairs_to_drop = set()
    cols = df.columns
    for i in range(0, df.shape[1]):
        for j in range(0, i+1):
            pairs_to_drop.add((cols[i], cols[j]))
    return pairs_to_drop

def get_top_abs_correlations(df, n=5):
    au_corr = df.corr().abs().unstack()
    labels_to_drop = get_redundant_pairs(df)
    au_corr = au_corr.drop(labels=labels_to_drop).sort_values(ascending=False)
    return au_corr[0:n]

print("Top Absolute Correlations")
print(get_top_abs_correlations(X_train, 5))<jupyter_output>Top Absolute Correlations
32   75     0.259315
101  193    0.252825
22   28     0.246062
140  187    0.244012
79   187    0.243916
dtype: float64<jupyter_text>As we can see the features have no correlation  Baseline Linear Model (logistic regression)<jupyter_code>cont_scale_pipeline = make_pipeline(SimpleImputer(strategy = "median"), StandardScaler())
pipe_scale = make_pipeline(cont_scale_pipeline, LogisticRegression())
scores_pipe_scale = cross_val_score(pipe_scale, X_train, y_train, cv=3, scoring='roc_auc')
model = pipe_scale.fit(X_train, y_train)
pred = model.predict(X_test)
print("The auc-roc scores are ",scores_pipe_scale)
genop(pred)<jupyter_output>The auc-roc scores are  [0.72962963 0.71069182 0.65849057]<jupyter_text>This model got a public score of 0.666 on Kaggle  Baseline Linear Model (Linear SVC)<jupyter_code>cont_scale_pipeline = make_pipeline(SimpleImputer(strategy = "median"), StandardScaler())
pipe_scale = make_pipeline(cont_scale_pipeline, LinearSVC())
scores_pipe_scale = cross_val_score(pipe_scale, X_train, y_train, cv=3, scoring='roc_auc')
model = pipe_scale.fit(X_train, y_train)
pred = model.predict(X_test)
print("The auc-roc scores are ",scores_pipe_scale)
genop(pred)<jupyter_output><empty_output><jupyter_text>This model got a public score of 0.656 on Kaggle  Feature Selection   SelectKbest technique (K=35)<jupyter_code>select = SelectKBest(k=35, score_func=f_classif)
select.fit(X_train, y_train)
print(X_train.shape)
print(select.transform(X_train).shape)
up_X_train = pd.DataFrame(select.transform(X_train))
up_X_test = pd.DataFrame(select.transform(X_test))


cont_scale_pipeline = make_pipeline(SimpleImputer(strategy = "median"), StandardScaler())
pipe_scale = make_pipeline(cont_scale_pipeline, LogisticRegression(C=0.1))
scores_pipe_scale = cross_val_score(pipe_scale, X_train, y_train, cv=3, scoring='roc_auc')
model = pipe_scale.fit(up_X_train, y_train)
pred = model.predict(up_X_test)
print("The auc-roc scores are ",scores_pipe_scale)
genop(pred)<jupyter_output><empty_output><jupyter_text>This model got a public score of 0.722 on Kaggle  SelectKbest technique (K=75)<jupyter_code>select = SelectKBest(k=75, score_func=f_classif)
select.fit(X_train, y_train)
print(X_train.shape)
print(select.transform(X_train).shape)
up_X_train = pd.DataFrame(select.transform(X_train))
up_X_test = pd.DataFrame(select.transform(X_test))


cont_scale_pipeline = make_pipeline(SimpleImputer(strategy = "median"), StandardScaler())
pipe_scale = make_pipeline(cont_scale_pipeline, LogisticRegression(C=0.1))
scores_pipe_scale = cross_val_score(pipe_scale, X_train, y_train, cv=3, scoring='roc_auc')
model = pipe_scale.fit(up_X_train, y_train)
pred = model.predict(up_X_test)
print("The auc-roc scores are ",scores_pipe_scale)
genop(pred)<jupyter_output><empty_output><jupyter_text>This model got a public score of 0.706 on Kaggle  SelectPercentile(10)<jupyter_code>select = SelectPercentile(percentile=10, score_func=f_classif)
select.fit(X_train, y_train)
print(X_train.shape)
print(select.transform(X_train).shape)
up_X_train = pd.DataFrame(select.transform(X_train))
up_X_test = pd.DataFrame(select.transform(X_test))


cont_scale_pipeline = make_pipeline(SimpleImputer(strategy = "median"), StandardScaler())
pipe_scale = make_pipeline(cont_scale_pipeline, LogisticRegression(C=0.1))
scores_pipe_scale = cross_val_score(pipe_scale, X_train, y_train, cv=3, scoring='roc_auc')
model = pipe_scale.fit(up_X_train, y_train)
pred = model.predict(up_X_test)
print("The auc-roc scores are ",scores_pipe_scale)
genop(pred)<jupyter_output><empty_output><jupyter_text>The model got a public score of 0.720 on Kaggle  Mutual_info_classif<jupyter_code>scores = mutual_info_classif(X_train, y_train)
up_X_train = X_train.iloc[:,list(np.nonzero(scores)[0])]
up_X_test = X_test.iloc[:,list(np.nonzero(scores)[0])]


cont_scale_pipeline = make_pipeline(SimpleImputer(strategy = "median"), StandardScaler())
pipe_scale = make_pipeline(cont_scale_pipeline, LogisticRegression(C=0.1))
scores_pipe_scale = cross_val_score(pipe_scale, X_train, y_train, cv=3, scoring='roc_auc')
model = pipe_scale.fit(up_X_train, y_train)
pred = model.predict(up_X_test)
print("The auc-roc scores are ",scores_pipe_scale)
genop(pred)<jupyter_output><empty_output><jupyter_text>The model got a public score of 0.653 on Kaggle  SelectFromModel<jupyter_code>select= SelectFromModel(LogisticRegression())
select.fit(X_train, y_train)
print(select.transform(X_train).shape)
up_X_train = pd.DataFrame(select.transform(X_train))
up_X_test = pd.DataFrame(select.transform(X_test))


cont_scale_pipeline = make_pipeline(SimpleImputer(strategy = "median"), StandardScaler())
pipe_scale = make_pipeline(cont_scale_pipeline, LogisticRegression(C=0.1))
scores_pipe_scale = cross_val_score(pipe_scale, X_train, y_train, cv=3, scoring='roc_auc')
model = pipe_scale.fit(up_X_train, y_train)
pred = model.predict(up_X_test)
print("The auc-roc scores are ",scores_pipe_scale)
genop(pred)<jupyter_output><empty_output><jupyter_text>The model got a public score of 0.673 on Kaggle  RFE (Recursive Feature Elimination)<jupyter_code>rfe = RFE(LinearSVC(max_iter=10000), n_features_to_select=10)
rfe.fit(X_train, y_train)
scores_pipe_scale = cross_val_score(rfe, X_train, y_train, cv=3, scoring='roc_auc')
print("The auc-roc scores are ",scores_pipe_scale)
pred = rfe.predict(X_test)
genop(pred)<jupyter_output><empty_output><jupyter_text>The model got a public score of 0.719 on Kaggle  The best feature selection from the above analysis is the SelectKbest. The next piece of analysis is to run different classification models with SelectKbest feature selection<jupyter_code>select = SelectKBest(k=35, score_func=f_classif)
select.fit(X_train, y_train)
print(X_train.shape)
print(select.transform(X_train).shape)
up_X_train = pd.DataFrame(select.transform(X_train))
up_X_test = pd.DataFrame(select.transform(X_test))


cont_scale_pipeline = make_pipeline(SimpleImputer(strategy = "median"), StandardScaler())<jupyter_output><empty_output><jupyter_text>Linear SVC<jupyter_code>pipe_scale = make_pipeline(cont_scale_pipeline, LinearSVC(C=0.1))
scores_pipe_scale = cross_val_score(pipe_scale, X_train, y_train, cv=3, scoring='roc_auc')
model = pipe_scale.fit(up_X_train, y_train)
pred = model.predict(up_X_test)
print("The auc-roc scores are ",scores_pipe_scale)
genop(pred)<jupyter_output><empty_output><jupyter_text>The model got a public score of 0.731 on Kaggle  Decision Tree Classifier<jupyter_code>pipe_scale = make_pipeline(cont_scale_pipeline, DecisionTreeClassifier())
scores_pipe_scale = cross_val_score(pipe_scale, X_train, y_train, cv=3, scoring='roc_auc')
model = pipe_scale.fit(up_X_train, y_train)
pred = model.predict(up_X_test)
print("The auc-roc scores are ",scores_pipe_scale)
genop(pred)<jupyter_output><empty_output><jupyter_text>The model got a public score of 0.591 on Kaggle  Extra Tree Classifier<jupyter_code>pipe_scale = make_pipeline(cont_scale_pipeline, ExtraTreeClassifier())
scores_pipe_scale = cross_val_score(pipe_scale, X_train, y_train, cv=3, scoring='roc_auc')
model = pipe_scale.fit(up_X_train, y_train)
pred = model.predict(up_X_test)
print("The auc-roc scores are ",scores_pipe_scale)
genop(pred)<jupyter_output><empty_output><jupyter_text>The model got a public score of 0.551 on Kaggle  Random Forest  Classifier<jupyter_code>pipe_scale = make_pipeline(cont_scale_pipeline, RandomForestClassifier())
scores_pipe_scale = cross_val_score(pipe_scale, X_train, y_train, cv=3, scoring='roc_auc')
model = pipe_scale.fit(up_X_train, y_train)
pred = model.predict(up_X_test)
print("The auc-roc scores are ",scores_pipe_scale)
genop(pred)<jupyter_output><empty_output><jupyter_text>The model got a public score of 0.635 on Kaggle  Ada BoostClassifier<jupyter_code>pipe_scale = make_pipeline(cont_scale_pipeline, AdaBoostClassifier())
scores_pipe_scale = cross_val_score(pipe_scale, X_train, y_train, cv=3, scoring='roc_auc')
model = pipe_scale.fit(up_X_train, y_train)
pred = model.predict(up_X_test)
print("The auc-roc scores are ",scores_pipe_scale)
genop(pred)<jupyter_output><empty_output><jupyter_text>The model got a public score of 0.670 on Kaggle  Gradient BoostClassifier<jupyter_code>pipe_scale = make_pipeline(cont_scale_pipeline, GradientBoostingClassifier())
scores_pipe_scale = cross_val_score(pipe_scale, X_train, y_train, cv=3, scoring='roc_auc')
model = pipe_scale.fit(up_X_train, y_train)
pred = model.predict(up_X_test)
print("The auc-roc scores are ",scores_pipe_scale)
genop(pred)<jupyter_output><empty_output><jupyter_text>The model got a public score of 0.679 on Kaggle  GLM<jupyter_code>select = SelectKBest(k=35, score_func=f_classif)
select.fit(X_train, y_train)
print(X_train.shape)
print(select.transform(X_train).shape)
up_X_train = pd.DataFrame(select.transform(X_train))
up_X_test = pd.DataFrame(select.transform(X_test))
model = sm.GLM(y_train, up_X_train,family=sm.families.Binomial())
model_results = model.fit()
pred = model_results.predict(up_X_test)
genop(pred)<jupyter_output><empty_output><jupyter_text>The model got a public score of 0.731 on Kaggle<jupyter_code>select = SelectKBest(k=15, score_func=f_classif)
select.fit(X_train, y_train)
print(X_train.shape)
print(select.transform(X_train).shape)
up_X_train = pd.DataFrame(select.transform(X_train))
up_X_test = pd.DataFrame(select.transform(X_test))
model = sm.GLM(y_train, up_X_train,family=sm.families.Binomial())
model_results = model.fit()
pred = model_results.predict(up_X_test)
genop(pred)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1087771289113571, 'fraction_numerical': 0.01847816612917284, 'mean_word_length': 8.740237691001697, 'pattern_counts': {'":': 0, '<': 83, '<?xml version=': 0, '>': 83, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '21', 'dir': 'jupyter-structured-clean-dedup', 'id': '478614', 'license': 'MIT', 'n_tokens_mistral': 4344, 'n_tokens_neox': 4183, 'n_words': 761, 'path': 'Dont_Overfit.ipynb', 'repo_name': 'kskaran94/Dont-Overfit-Kaggle', 'size': '11473'}