<jupyter_start><jupyter_text>Load the dataset- Load the train data and using all your knowledge try to explore the different statistical properties of the dataset.<jupyter_code># Code starts here

data_train=pd.read_csv('/users/neha/Desktop/Pragra/Grey_Atom_Tasks/Sprint_8/Employee_Attrition_Prediction/train.csv')

data_train.head()

# Code ends here
data_train.info()
print('The null values in every feature are :\n', data_train.isnull().sum())
data_train.describe()
data_train.shape
data_train.loc[data_train['Attrition']=='Yes', 'Attrition'] = 1
data_train.loc[data_train['Attrition']=='No', 'Attrition'] = 0
categorical_columns = [c for c in data_train.columns 
                       if data_train[c].dtype.name == 'object']
numerical_columns = [c for c in data_train.columns 
                     if data_train[c].dtype.name != 'object']

print('categorical_columns:', categorical_columns)
print('numerical_columns:', numerical_columns)
data_train['Department'].value_counts()
data_train['HourlyRate'].value_counts()
data_train['Attrition'].value_counts()
data_train['MaritalStatus'].value_counts()
data_train['Gender'].value_counts()
num_cols = ['Age', 'DailyRate', 'DistanceFromHome', 'Education', 'EnvironmentSatisfaction', 'JobInvolvement', 'JobLevel', 'JobSatisfaction', 'MonthlyIncome', 'NumCompaniesWorked', 'PercentSalaryHike', 'PerformanceRating', 'RelationshipSatisfaction', 'StandardHours', 'StockOptionLevel', 'TotalWorkingYears', 'TrainingTimesLastYear', 'WorkLifeBalance','YearsAtCompany','YearsInCurrentRole', 'YearsSinceLastPromotion', 'YearsWithCurrManager']
for i in range(0,len(num_cols),2):
    if len(num_cols) > i+1:
        plt.figure(figsize=(10,4))
        plt.subplot(121)
        sns.distplot(data_train[num_cols[i]], kde=False)
        plt.subplot(122)            
        sns.distplot(data_train[num_cols[i+1]], kde=False)
        plt.tight_layout()
        plt.show()

    else:
        sns.distplot(data_train[num_cols[i]], kde=False)
sns.scatterplot(data=data_train, x= "Attrition",y="Age", hue="Gender")
markers = {"Male": "s", "Female": "X"}
sns.scatterplot(data=data_train, x="TotalWorkingYears", y='YearsAtCompany', style="Gender",hue='Attrition', markers=markers)
sns.scatterplot(data=data_train, x="Age", y="WorkLifeBalance", hue="MaritalStatus", palette="deep")
data_train.corr()
plt.figure(figsize=[15,8])
heatmap = sns.heatmap(data_train.corr(), vmin=-1, vmax=1,annot=True,fmt='.2f', cmap='BrBG');
heatmap.set_title('Correlation Heatmap', fontdict={'fontsize':20}, pad=20);
data_train.drop(['Id','EmployeeNumber','Over18','EmployeeCount','StandardHours'],1, inplace=True)
categorical_columns = [c for c in data_train.columns 
                       if data_train[c].dtype.name == 'object']

for column in categorical_columns:
    data_train[column]=data_train[column].astype('category').cat.codes
    
print(data_train.info())<jupyter_output><class 'pandas.core.frame.DataFrame'>
RangeIndex: 1281 entries, 0 to 1280
Data columns (total 31 columns):
 #   Column                    Non-Null Count  Dtype
---  ------                    --------------  -----
 0   Age                       1281 non-null   int64
 1   BusinessTravel            1281 non-null   int8 
 2   DailyRate                 1281 non-null   int64
 3   Department                1281 non-null   int8 
 4   DistanceFromHome          1281 non-null   int64
 5   Education                 1281 non-null   int64
 6   EducationField            1281 non-null   int8 
 7   EnvironmentSatisfaction   1281 non-null   int64
 8   Gender                    1281 non-null   int8 
 9   HourlyRate                1281 non-null   int64
 10  JobInvolvement            1281 non-null   int64
 11  JobLevel                  1281 non-null   int64
 12  JobRole                   1281 non-null   int8 
 13  JobSatisfaction           1281 non-null   int64
 14  MaritalStatus             1281 non-null [...]<jupyter_text>Model building- Now let's come to the actual task, using Decision Tree/Ensemble Technique, predict the `Attrition`. Use different techniques you have learned to imporove the performance of the model.- Try improving upon the [ROC-AUC Score](https://scikit-learn.org/stable/modules/generated/sklearn.metrics.roc_auc_score.html)<jupyter_code>X=data_train.drop(['Attrition'], axis=1)
y=data_train['Attrition'].copy()
import scipy.stats as scs
# Code starts here

features = []
skewness = []
for i in X.select_dtypes(include=np.number).columns:
    features.append(i)
    skewness.append(scs.skew(X[i]))
skewed = pd.DataFrame({'Features':features,'Skewness':skewness})
# If skewness is greater than 1 the feature is highly positively skewed
positively_skewed_variables = skewed[(skewed['Skewness']>1)]
# If the skewness is less than -1 the feature is highly negatively skewed.
negatively_skewed_variables = skewed[(skewed['Skewness']<-1)]

print('Positively Skewed Features \n',positively_skewed_variables)
print('*'*50)
print('Negatively Skewed Features \n',negatively_skewed_variables) 

# Let's remove the skewness in the positively skewed variables by using a log transform
for i in positively_skewed_variables['Features']:
    X[i] = np.log1p(X[i])
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=0)<jupyter_output><empty_output><jupyter_text>Logistic Regression<jupyter_code>logreg = LogisticRegression()
logreg.fit(X_train,y_train)
y_pred = logreg.predict(X_test)
print('The AUC_ROC_score for a logistic regression model is :' ,roc_auc_score(y_test,y_pred))
print('The accuracy for logistic regression is : ', accuracy_score(y_test,y_pred))<jupyter_output>The AUC_ROC_score for a logistic regression model is : 0.5294576329925489
The accuracy for logistic regression is :  0.7821011673151751<jupyter_text>Decision Tree Classifier<jupyter_code>tree = DecisionTreeClassifier(max_depth=3, random_state=17)
tree.fit(X_train, y_train)
tree_predictions = tree.predict(X_test) 
print("Decision tree accuracy: ",accuracy_score(y_test,tree_predictions))
print('The AUC_ROC_score for a Decision tree model is :' ,roc_auc_score(y_test,tree_predictions))
tree_params = {'max_depth': range(2, 11)}
locally_best_tree = GridSearchCV(DecisionTreeClassifier(random_state=17),tree_params, cv=5)                  
locally_best_tree.fit(X_train, y_train)
print("Best params:", locally_best_tree.best_params_)
print("Best cross validaton score", locally_best_tree.best_score_)
tuned_tree = DecisionTreeClassifier(max_depth=2, random_state=17)
tuned_tree.fit(X_train, y_train)
tuned_tree_predictions = tuned_tree.predict(X_test)
print("Decision tree Accuracy after tuning: ",accuracy_score(y_test, tuned_tree_predictions))
print('The AUC_ROC_score for a Tuned Decision tree model is :' ,roc_auc_score(y_test,tuned_tree_predictions))<jupyter_output>Best params: {'max_depth': 5}
Best cross validaton score 0.7998278335724534
Decision tree Accuracy after tuning:  0.7782101167315175
The AUC_ROC_score for a Tuned Decision tree model is : 0.5574857043839889<jupyter_text>Next let's implement a Stacking Classifier consisting of one Decision Tree and Logistic regression and a decision tree model as a meta classifier.<jupyter_code>from mlxtend.classifier import StackingClassifier

classifier1 = LogisticRegression(C=0.1)
classifier2= DecisionTreeClassifier(random_state=2,criterion='entropy',max_depth=4)

# classifier
classifier_list=[classifier1,classifier2]

# meta classifier
m_classifier=DecisionTreeClassifier(random_state=42,criterion='entropy',max_depth=6)

# Code starts here

sclf = StackingClassifier(classifiers=classifier_list, meta_classifier=m_classifier)

sclf.fit(X_train,y_train)

y_pred_sclf = sclf.predict(X_test)
# s_score=sclf.score(X_train,y_train)
# print("Training score: %.2f " % s_score)

s_score=sclf.score(X_test,y_test)
print("Test score: %.2f " % s_score)
print('AUC_ROC_score for Stacking Classifier is ',roc_auc_score(y_test,y_pred))<jupyter_output>Test score: 0.80 
AUC_ROC_score for Stacking Classifier is  0.5294576329925489<jupyter_text>Random Forest Classifier<jupyter_code>from sklearn.ensemble import RandomForestClassifier

rfc = RandomForestClassifier(criterion='gini', max_depth= 3, max_features= 'auto',n_estimators=12)
# Fit the model
rfc.fit(X_train,y_train)
# Store the predicted values of test data
y_pred_rf = rfc.predict(X_test)
# accuracy
accuracy = rfc.score(X_test,y_test)
# roc score
roc_score = roc_auc_score(y_test,y_pred_rf)
print('Accuracy is ', accuracy_score(y_test,y_pred_rf))
print('AUC_ROC score for Random Forest is: ',roc_score)
print(classification_report(y_test,y_pred_rf))<jupyter_output>Accuracy is  0.7898832684824902
AUC_ROC score for Random Forest is:  0.5466990123028937
              precision    recall  f1-score   support

           0       0.79      0.99      0.88       199
           1       0.75      0.10      0.18        58

    accuracy                           0.79       257
   macro avg       0.77      0.55      0.53       257
weighted avg       0.78      0.79      0.72       257<jupyter_text>Let's try improving upon the accuracy of the Random Forest using hyperparameter tuning<jupyter_code>rfc = RandomForestClassifier(random_state=2)
param_grid = { 
'n_estimators': [11,12,13],
'max_features': ['auto', 'sqrt', 'log2'],
'max_depth' : [4,5,6,7,8],
'criterion' :['gini', 'entropy']
}
# grid search cv

grid_search_model = GridSearchCV(rfc, param_grid=param_grid)
grid_search_model.fit(X_train, y_train)

grid_search_model.best_params_
from sklearn.ensemble import RandomForestClassifier

# Initialize RandomForrest model to variable rfc
rfc = RandomForestClassifier(criterion='entropy', max_depth= 8,max_features= 'auto', n_estimators= 12)

# Fit the model
rfc.fit(X_train,y_train)

# Store the predicted values of test data
y_pred_rf = rfc.predict(X_test)

# accuracy
accuracy = rfc.score(X_test,y_test)

# roc score
roc_score = roc_auc_score(y_test,y_pred_rf)

print('AUC_ROC score is ',roc_score)
print('Accuracy is ', accuracy_score(y_test,y_pred_rf))
print(classification_report(y_test,y_pred_rf))<jupyter_output>AUC_ROC score is  0.7582741292670246
Accuracy is  0.8715953307392996
              precision    recall  f1-score   support

           0       0.88      0.96      0.92       199
           1       0.82      0.55      0.66        58

    accuracy                           0.87       257
   macro avg       0.85      0.76      0.79       257
weighted avg       0.87      0.87      0.86       257<jupyter_text>Now let's apply a Voting Classifier that consists of three models - Logistic Regression, Decision Tree and a Random Forest with a soft voting<jupyter_code>from sklearn.ensemble import VotingClassifier

# Various models
clf_1 = LogisticRegression()
clf_2 = DecisionTreeClassifier(random_state=4)
clf_3 = RandomForestClassifier(random_state=4)

model_list = [('lr',clf_1),('DT',clf_2),('RF',clf_3)]
# Initialize voting classifier
voting_clf_hard = VotingClassifier(estimators=model_list,voting='soft')
# Fit the model on training data
voting_clf_hard.fit(X_train,y_train)
# predict on test
y_pred_vot = voting_clf_hard.predict(X_test)
# accuracy
hard_voting_score = voting_clf_hard.score(X_test,y_test)

print("AUC-ROC score for Voting Classifier is:", roc_auc_score(y_test,y_pred_vot))
print("Accuracy score for Voting Classifier is:", accuracy_score(y_test,y_pred_vot))
print("Classification Report for Voting Classifier is:",classification_report(y_test,y_pred_vot))<jupyter_output>AUC-ROC score for Voting Classifier is: 0.8096517068099117
Accuracy score for Voting Classifier is: 0.8754863813229572
Classification Report for Voting Classifier is:               precision    recall  f1-score   support

           0       0.91      0.93      0.92       199
           1       0.74      0.69      0.71        58

    accuracy                           0.88       257
   macro avg       0.83      0.81      0.82       257
weighted avg       0.87      0.88      0.87       257<jupyter_text>Let's check how a bagging classifier using a Decision Tree performs on our data<jupyter_code># Import Bagging Classifier
from sklearn.ensemble import BaggingClassifier
# Initialize Bagging Classifier
bagging_clf = BaggingClassifier(DecisionTreeClassifier(), random_state=0,n_estimators=100,max_samples=100)
# Fit the model on training data
bagging_clf.fit(X_train,y_train)
# Predicted values of X_test
y_pred_bagging = bagging_clf.predict(X_test)
# accuracy 
print('Accuracy for Bagging Classifier is ',bagging_clf.score(X_test,y_test))
# roc_score
score_bagging = roc_auc_score(y_test,y_pred_bagging)
print('AUC_ROC_score for bagging classifier is ',score_bagging)
print(classification_report(y_test,y_pred_bagging))<jupyter_output>Accuracy for Bagging Classifier is  0.7782101167315175
AUC_ROC_score for bagging classifier is  0.5574857043839889
              precision    recall  f1-score   support

           0       0.80      0.96      0.87       199
           1       0.53      0.16      0.24        58

    accuracy                           0.78       257
   macro avg       0.66      0.56      0.56       257
weighted avg       0.74      0.78      0.73       257<jupyter_text>Prediction on the test data and creating the sample submission file.- Load the test data and store the `Id` column in a separate variable.- Perform the same operations on the test data that you have performed on the train data.- Create the submission file as a `csv` file consisting of the `Id` column from the test data and your prediction as the second column.<jupyter_code># Code Starts here
data_test=pd.read_csv('/users/neha/Desktop/Pragra/Grey_Atom_Tasks/Sprint_8/Employee_Attrition_Prediction/test.csv')
data_test.shape


data_test.drop(['EmployeeNumber','Over18','EmployeeCount','StandardHours'],1, inplace=True)

# Code ends here
categorical_columns = [c for c in data_test.columns 
                       if data_test[c].dtype.name == 'object']

for column in categorical_columns:
    data_test[column]=data_train[column].astype('category').cat.codes
    
print(data_test.info())
# Storing the id from the test file
id_ = data_test['Id']

# Dropping the same columns from the test data
data_test.drop(['Id'],1,inplace=True)

# Applying rfe on test data

y_pred_test= rfc.predict(data_test)

# Create a sample submission file
final_submission = pd.DataFrame({'Id':id_,'Attrition':y_pred_test})

final_submission.loc[final_submission['Attrition']==1, 'Attrition'] = 'Yes'
final_submission.loc[final_submission['Attrition']==0, 'Attrition'] = 'No'

# Convert the sample submission file into a csv file
final_submission.to_csv('final_submission.csv',index=False)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.10200451529041527, 'fraction_numerical': 0.046931654922350685, 'mean_word_length': 4.218850410567654, 'pattern_counts': {'":': 2, '<': 35, '<?xml version=': 0, '>': 36, 'https://': 1, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 16, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '10', 'dir': 'jupyter-structured-clean-dedup', 'id': '521631', 'license': 'MIT', 'n_tokens_mistral': 5350, 'n_tokens_neox': 4852, 'n_words': 1110, 'path': 'Employee_Attrition_Student_Template.ipynb', 'repo_name': 'Neha15pant/employee-attrition-prediction', 'size': '14617'}