<jupyter_start><jupyter_text><jupyter_code># !pip install pandas-profiling
import numpy as np
import pandas as pd
from pandas_profiling import ProfileReport
import matplotlib.pyplot as plt
import seaborn as sns
import sklearn
import scipy
import pandas.util.testing as tm
train = pd.read_csv('train.csv')
print(train.shape)
train.head()
train.describe()
# making a bool series 
bool_series = train['Loan.ID'].duplicated(keep = False) 
  
# bool series 
bool_series 
  
# passing NOT of bool series to see unique values only 
train = train[~bool_series] 

print(train.shape)

train.head()
# sns.set(style="whitegrid")
# sns.boxplot(x=train['Current.Loan.Amount'])

plt.hist(x=train['Current.Loan.Amount'])
train['Current.Loan.Amount'] = train['Current.Loan.Amount'].apply(np.log)
# sns.set(style="whitegrid")
# plt.figure()
# sns.boxplot(x=train['Current.Loan.Amount'])
plt.figure()
plt.hist(x=train['Current.Loan.Amount'])
train.head()
plt.figure()
sns.heatmap(train.isnull(),yticklabels=False,cbar=False,cmap='viridis')
train.drop(['Months.since.last.delinquent'], axis=1, inplace=True)
train.shape
print(np.mean(train['Credit.Score']))
mean_credit_score = np.mean(train['Credit.Score'])

y=train['Credit.Score']
train['Loan.ID'].tail()

# plt.scatter(x, y, linewidths=0.1)
train['Credit.Score'].fillna(mean_credit_score, inplace=True)
train.dropna(subset=['Years.in.current.job'], inplace=True)
train.shape
plt.figure()
sns.heatmap(train.isnull(),yticklabels=False,cbar=False,cmap='viridis')
indexNames = train[train['Annual.Income'] > 800000 ].index
train.drop(indexNames , inplace=True, axis=0)
print(train.shape)
sns.boxplot(x=train['Annual.Income'])
mean_annual_income=np.mean(train['Annual.Income'])
train['Annual.Income'].fillna(mean_annual_income, inplace=True)
mean_Bankruptcies=np.mean(train['Bankruptcies'])
train['Bankruptcies'].fillna(mean_Bankruptcies, inplace=True)
plt.figure()
sns.heatmap(train.isnull(),yticklabels=False,cbar=False,cmap='viridis')
sns.boxplot(x=train['Credit.Score'])

train['Credit.Score'] = train['Credit.Score'].apply(np.log)
# plt.hist(x=train['Credit.Score'])

sns.boxplot(x=train['Credit.Score'])
print(train.shape)
sns.boxplot(x=train['Annual.Income'])
# len(train[train['Annual.Income'] == 0])
# sns.boxplot(x=train['Annual.Income'])
# print(train.shape)
# # for i in range(len(train)):
# #   if train['Annual.Income'].index[i] > 800000:
# #     train.drop(index=i, inplace=True, axis=0)
# train['Gross.Monthly.Income'] = train['Gross.Monthly.Income'].apply(np.log)
# plt.figure()
# sns.boxplot(x=train['Gross.Monthly.Income'])

train.head()
sns.boxplot(x=train['Years.of.Credit.History'])
# len(train[train['Years.of.Credit.History']==0])
train['Years.of.Credit.History'] = train['Years.of.Credit.History'].apply(np.log)
sns.boxplot(x=train['Years.of.Credit.History'])
ac = train['Maximum.Open.Credit'].values-train['Current.Credit.Balance'].values
train.insert(15, column='Available.Credits', value=ac)
train.drop(columns=['Current.Credit.Balance', 'Maximum.Open.Credit'], axis=1, inplace=True)

train.head()
from sklearn.preprocessing import LabelEncoder
X=train
categorical_feature_mask = X.dtypes==object

categorical_cols = X.columns[categorical_feature_mask].tolist()

le = LabelEncoder()
X[categorical_cols] = X[categorical_cols].apply(lambda col: le.fit_transform(col))+1
train = X
train.head()
mean_annual_income=np.mean(train['Tax.Liens'])
train['Tax.Liens'].fillna(mean_annual_income, inplace=True)
EMI = train['Current.Loan.Amount']/train['Term']
train.insert(0, column='EMI', value=EMI)
train.drop(columns=['Current.Loan.Amount', 'Term'], axis=1, inplace=True)

train.head()
y_train = train['Loan.Status']

ID_train = train['Loan.ID']

train.drop(['Loan.Status', 'Loan.ID'], axis=1, inplace=True)
X_train = train

X_train.head()
from sklearn.preprocessing import StandardScaler

scaler = StandardScaler()
scaler.fit(X_train)

X_train = pd.DataFrame(scaler.transform(X_train), columns=X_train.columns)

X_train.head()
# from sklearn.preprocessing import RobustScaler

# scaler = RobustScaler()
# scaler.fit(X_train)
# X_train = pd.DataFrame(scaler.transform(X_train), columns=X_train.columns)

# X_train.head()
np.isfinite(train).all()
from sklearn.model_selection import train_test_split as tts

X_train, X_validation, y_train, y_validation = tts(X_train, y_train, train_size=0.75, test_size=0.25)
X_train.head()
# from sklearn.ensemble import ExtraTreesClassifier
# from sklearn.feature_selection import SelectFromModel
# print(X_train.shape)
# clf = ExtraTreesClassifier(n_estimators=50)
# clf = clf.fit(X_train, y_train)
# col = clf.feature_importances_ 
# model = SelectFromModel(clf, prefit=True)
# feature_idx = model.get_support()
# feature_name = X_train.columns[feature_idx]
# X_train = pd.DataFrame(model.transform(X_train), columns=feature_name)
# print(X_train.shape)   
# X_train.head()
# X_validation = X_validation.loc[:, feature_name]
# X_validation.head()
# import imblearn
# from imblearn.under_sampling import RandomUnderSampler
# import xgboost as xgb
# from sklearn.ensemble import RandomForestClassifier
# from sklearn import svm
# from sklearn.linear_model import LogisticRegression
# from sklearn.ensemble import VotingClassifier


# random_undersampler = RandomUnderSampler()
# X_res, y_res = random_undersampler.fit_sample(X_train, y_train)

# # Shuffle the data
# perms = np.random.permutation(X_res.shape[0])
# X_res = X_res[perms]
# y_res = y_res[perms]

# # Define base learners
# xgb_clf = xgb.XGBClassifier(objective="binary:logistic",
#                             learning_rate=0.03,
#                             n_estimators=500,
#                             max_depth=1,
#                             subsample=0.4,
#                             random_state=123)
# svm_clf = svm.SVC(gamma=0.1,
#               C=0.01,
#               kernel="poly",
#               degree=3,
#               coef0=10.0,
#               probability=True)
# rf_clf = RandomForestClassifier(n_estimators=300,
#                                 max_features="sqrt",
#                                 criterion="gini",
#                                 min_samples_leaf=5,
#                                 class_weight="balanced")

# # Define meta-learner
# logreg_clf = LogisticRegression(penalty="l2", C=100, fit_intercept=True)
# # Fitting voting clf --> average ensemble
# voting_clf = VotingClassifier([("xgb", xgb_clf),
#                                ("svm", svm_clf),
#                                ("rf", rf_clf)],
#                               voting="soft",
#                               flatten_transform=True)
# voting_clf.fit(X_res, y_res)
# from sklearn.pipeline import make_pipeline
# from sklearn.preprocessing import FunctionTransformer
# from sklearn.model_selection import cross_val_predict

# xgb_model, svm_model, rf_model = voting_clf.estimators_
# models = {"xgb": xgb_model,
#           "svm": svm_model,
#           "rf": rf_model,
#           "avg_ensemble": voting_clf}

# # Build first stack of base learners
# first_stack = make_pipeline(voting_clf, FunctionTransformer(lambda X: X[:, 1::2]))

# # Use CV to generate meta-features
# meta_features = cross_val_predict(first_stack, X_res, y_res, cv=10, method="transform")

# # Refit the first stack on the full training set
# first_stack.fit(X_res, y_res)

# # Fit the meta learner
# second_stack = logreg_clf.fit(meta_features, y_res)

# stacked_probs = second_stack.predict(first_stack.transform(X_train.values))
# stacked_probs
# stacked_auc_score = roc_auc_score(y_validation.values, stacked_probs)
# from sklearn.neural_network import MLPClassifier
# clf = MLPClassifier(solver='adam', activation='tanh', alpha=0.0001, max_iter=10000, hidden_layer_sizes=(150,150,50), random_state=1)
# clf.fit(X_train, y_train)
# from sklearn.naive_bayes import GaussianNB

# model = GaussianNB()
# model.fit(X_train, y_train)
# import tensorflow as tf

# model = tf.keras.Sequential([
#             tf.keras.layers.Flatten(),
#             tf.keras.layers.Dense(100, activation='relu'),
#             tf.keras.layers.Dense(100, activation='relu'),
#             tf.keras.layers.Dense(50, activation='relu'),
#             tf.keras.layers.Dropout(0.5),
#             tf.keras.layers.Dense(2, activation='softmax')
# ])

# model.compile(loss='binary_crossentropy',
#               optimizer='adam',
#               metrics=['accuracy']
#               )

# history=model.fit(
#   X_train,
#   y_train,
#   steps_per_epoch=10000,
#   epochs=50,
#   validation_data=(X_validation, y_validation)
#   )


# model.summary()
# from sklearn.neighbors import KNeighborsClassifier

# model = KNeighborsClassifier(n_neighbors=2, metric='minkowski', weights='distance')

# model.fit(X_train, y_train)
# from sklearn import svm

# model = svm.SVC()
# model.fit(X_train, y_train)
# from sklearn.ensemble import ExtraTreesClassifier

# model = ExtraTreesClassifier()
# model.fit(X_train,y_train)
from sklearn.ensemble import RandomForestClassifier
from sklearn.ensemble import BaggingClassifier
from sklearn.tree import DecisionTreeClassifier
from sklearn.ensemble import VotingClassifier
from sklearn.linear_model import LogisticRegression
from sklearn import svm

# model = RandomForestClassifier(random_state = 1, max_depth = 15, n_estimators = 500, min_samples_split = 2, min_samples_leaf = 1)
model = LogisticRegression(penalty='elasticnet', solver='saga', multi_class='multinomial', max_iter=300, C=1, l1_ratio=0.8)
# sv = svm.SVC(kernel='poly', degree=3, C=1)

# model = VotingClassifier(estimators=([('svc', sv), ('dt', dtc), ('lr', lr)]), voting='hard')
model.fit(X_train,y_train)
# from sklearn.ensemble import AdaBoostClassifier
# from sklearn.ensemble import GradientBoostingClassifier
# from sklearn.ensemble import RandomForestClassifier

# model = GradientBoostingClassifier(RandomForestClassifier(random_state = 1, max_depth = 15, n_estimators = 500, min_samples_split = 2, min_samples_leaf = 1), n_estimators=40, learning_rate=0.1)
# model = GradientBoostingClassifier(DecisionTreeClassifier())
# model.fit(X_train, y_train)
# from sklearn.naive_bayes import BernoulliNB
# multiNB = BernoulliNB(binarize=0.1)

# model = multiNB.fit(X_train, y_train)
# from sklearn.model_selection import GridSearchCV

# n_estimators = [100, 300, 500, 800, 1200]
# max_depth = [5, 8, 15, 25, 30]
# min_samples_split = [2, 5, 10, 15, 100]
# min_samples_leaf = [1, 2, 5, 10] 

# hyperF = dict(n_estimators = n_estimators, max_depth = max_depth,  
#               min_samples_split = min_samples_split, 
#              min_samples_leaf = min_samples_leaf)

# gridF = GridSearchCV(RandomForestClassifier(random_state = 1, max_depth = 15, n_estimators = 500, min_samples_split = 2, min_samples_leaf = 1), hyperF, cv = 3, verbose = 1, n_jobs = -1)
# model = gridF.fit(X_train, y_train)
# N = 20
# plt.style.use("ggplot")
# plt.figure()
# plt.plot(np.arange(0, N), history.history["loss"], label="train_loss")
# plt.plot(np.arange(0, N), history.history["val_loss"], label="val_loss")
# plt.plot(np.arange(0, N), history.history["accuracy"], label="train_acc")
# plt.plot(np.arange(0, N), history.history["val_accuracy"], label="val_acc")
# plt.title("Training Loss and Accuracy")
# plt.xlabel("Epoch #")
# plt.ylabel("Loss/Accuracy")
# plt.legend(loc="lower left")
from sklearn.metrics import r2_score
from sklearn.metrics import f1_score
from sklearn.metrics import accuracy_score

Y_predict_training = model.predict(X_train)
# stacked_probs = second_stack.predict(first_stack.transform(X_validation.values))
# Y_predict_training=Y_predict_training+1
Y_predict_training
accuracy_score(y_train, Y_predict_training)
# y_train
f1_score(y_train, Y_predict_training)
Y_predict_validation = model.predict(X_validation)
# stacked_probs = second_stack.predict(first_stack.transform(X_validation.values))
accuracy_score(y_validation, Y_predict_validation)
X_train.shape
test = pd.read_csv('test.csv')

ID_test = test['Loan.ID']
test.drop(['Months.since.last.delinquent', 'Loan.ID', 'Loan.Status'], axis=1, inplace=True)

print(test.shape)

plt.figure()
sns.heatmap(test.isnull(),yticklabels=False,cbar=False,cmap='viridis')
mean_Bankruptcies=np.mean(test['Credit.Score'])
test['Credit.Score'].fillna(mean_Bankruptcies, inplace=True)

mean_Bankruptcies=np.mean(test['Bankruptcies'])
test['Bankruptcies'].fillna(mean_Bankruptcies, inplace=True)

mean_Bankruptcies=np.mean(test['Tax.Liens'])
test['Tax.Liens'].fillna(mean_Bankruptcies, inplace=True)

mean_Bankruptcies=np.mean(test['Annual.Income'])
test['Annual.Income'].fillna(mean_Bankruptcies, inplace=True)

test['Years.in.current.job'].fillna(test['Years.in.current.job'].mode()[0], inplace=True)

plt.figure()
sns.heatmap(test.isnull(),yticklabels=False,cbar=False,cmap='viridis')

print(test.shape)
test.head()
from sklearn.preprocessing import LabelEncoder
X=test
categorical_feature_mask = X.dtypes==object

categorical_cols = X.columns[categorical_feature_mask].tolist()

le = LabelEncoder()
X[categorical_cols] = X[categorical_cols].apply(lambda col: le.fit_transform(col))+1
test = X

test.head()
# gross_monthly_income = (test['Annual.Income']/12)-test['Monthly.Debt']
# test.insert(6, column='Gross.Monthly.Income', value=gross_monthly_income)
# test.drop(columns=['Monthly.Debt', 'Annual.Income'], axis=1, inplace=True)

# test['Gross.Monthly.Income'] = test['Gross.Monthly.Income'].apply(np.log)
test['Current.Loan.Amount'] = test['Current.Loan.Amount'].apply(np.log)

test['Credit.Score'] = test['Credit.Score'].apply(np.log)

test['Annual.Income'] = test['Annual.Income'].apply(np.log)

test['Years.of.Credit.History'] = test['Years.of.Credit.History'].apply(np.log)

ac = test['Maximum.Open.Credit'].values-test['Current.Credit.Balance'].values
ac=ac+1
test.insert(15, column='Available.Credits', value=ac)
test.drop(columns=['Current.Credit.Balance', 'Maximum.Open.Credit'], axis=1, inplace=True)

test['Available.Credits'] = test['Available.Credits']+0.5
len(test[test['Available.Credits'] == 0 ])

test.head()
EMI = test['Current.Loan.Amount']/test['Term']
test.insert(0, column='EMI', value=EMI)
test.drop(columns=['Current.Loan.Amount', 'Term'], axis=1, inplace=True)

test.head()
np.isfinite(test).all()
scaler = StandardScaler()
scaler.fit(test)
X_test = pd.DataFrame(scaler.transform(test), columns=test.columns)

X_test.head()
Y_predict_test = model.predict(X_test)
Y_predict_test = Y_predict_test
Y_predict_test
Y_test = pd.DataFrame({'Loan.ID': ID_test, 'Loan.Status': Y_predict_test})
Y_test.head(20)
Y_test.to_csv('Submission.csv', header=True, index=False)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.14998629010145326, 'fraction_numerical': 0.01473814093775706, 'mean_word_length': 5.307392996108949, 'pattern_counts': {'":': 4, '<': 5, '<?xml version=': 0, '>': 8, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '401997', 'license': 'Apache-2.0', 'n_tokens_mistral': 5478, 'n_tokens_neox': 5265, 'n_words': 816, 'path': 'Data_Crunch_Hackathon_20.ipynb', 'repo_name': 'Susheel06/Data-crunch-hackathon', 'size': '14588'}