<jupyter_start><jupyter_text>Absenteeism Data set<jupyter_code>#Load given Absenteeism data set into panda DataFrame
#Look a few records to get familiarized with the data
import pandas as pd
import numpy as np 
import matplotlib.pyplot as plt
from sklearn.model_selection import train_test_split
from sklearn import metrics
%matplotlib inline
df_original=pd.read_csv('./Absenteeism_at_work.csv',sep=';')
df_original.head()<jupyter_output><empty_output><jupyter_text>Data Preprocessing<jupyter_code>#Check all columns , their data types
df_original.info()
# Change data types of boolean and category columns appropriately
df = df_original.copy()
df['Social drinker'] = df['Social drinker'].astype('bool')
df['Social smoker'] = df['Social smoker'].astype('bool')
df['Disciplinary failure'] = df['Disciplinary failure'].astype('bool')
df['Seasons'] = df['Seasons'].astype('category')
df['Education'] = df['Education'].astype('category')
df['Day of the week'] = df['Day of the week'].astype('category')
df['Month of absence'] = df['Month of absence'].astype('category')
df['Reason for absence'] = df['Reason for absence'].astype('category')

df.info()
#Check for any missing values
df.isnull().sum()
#check for invalid data for e.g. Month of absence = 0
df[df['Month of absence']==0]
#Removing these records from dataframe
df = df[df['Month of absence'] != 0]
len(df)
len(df[df['Absenteeism time in hours']==0])
len(df[df['Disciplinary failure']==1])
# Since there is one record in disciplinary failure = 1 which has absenteeism time = 0.
df[(df['Absenteeism time in hours']==0) & (df['Disciplinary failure']!=1)]
# It looks like a valid record for employee id 34. We can replace absenteeism time with the mean value of same ID.
df[df['ID']==34][['Absenteeism time in hours']].mean()
#Update this record with mean value of 6
df.loc[(df['ID']==34) & (df['Absenteeism time in hours']==0),'Absenteeism time in hours']=6
#Verify update
df[(df['ID']==34) & (df['Absenteeism time in hours']==6)]

len(df[df['Disciplinary failure']==1])
#Update Absenteeism in hours as 8 for all the rows with Disciplinary failure assuming in 
#such situations, employees were asked to be absent
df.loc[(df['Absenteeism time in hours']==0),'Absenteeism time in hours']=8
len(df[df['Absenteeism time in hours']==0])
#drop weight and height attributes since BMI is a derived attribute which has 
#information about both these attributes
df=df.drop(['Weight'],axis=1)
df=df.drop(['Height'],axis=1)<jupyter_output><empty_output><jupyter_text>Exploratory Data Analysis<jupyter_code>#Mean of absenteeism time by seasons
import seaborn as sns
print(df['Absenteeism time in hours'].mean())
sns.jointplot(x='Absenteeism time in hours',y='Seasons',data=df)
#Absenteeism time by Age
#sns.jointplot(x='Age', y='Absenteeism time in hours',data=df)
age_abs = df.groupby('Age')[['Absenteeism time in hours']].mean()
ax = age_abs.plot(kind='bar', figsize=(10,6), legend=False)
for i, v in enumerate(age_abs.values):
    ax.text(i-.25, v + 1, str(np.int(np.round(v))), color='red')
ax.set_ylabel('Absenteeism time in hours')
ax.set_title('Average Absenteeism time in hours by age')
plt.show()
# Average work load per day by age
age_work_sum = df.groupby('Age', as_index=False)[['Work load Average/day ']].mean()
ax = age_work_sum.plot(kind='bar', x='Age', figsize=(8,6), legend=False)
ax.set_ylabel('Work load average per day')
ax.set_title('Average work load per day by age')
plt.show()
#Son and Pets by Age
ax = df.groupby('Age')['Son', 'Pet'].sum().plot(figsize=(8,6))
ax.set_ylabel('Count')
ax.set_title('Count of Pet & Son by Age')
plt.show()
#Absenteeism by Social driking and Smoking
drink_sum = df[(df['Social drinker'] == 1) & (df['Social smoker']== 0)]['Absenteeism time in hours'].sum()
smok_sum = df[(df['Social drinker'] == 0) & (df['Social smoker']== 1)]['Absenteeism time in hours'].sum()
drink_smok_sum = df[(df['Social drinker'] == 1) & (df['Social smoker'] == 1)]['Absenteeism time in hours'].sum()
abs_sum = df[(df['Social drinker'] == 0) & (df['Social smoker'] == 0)]['Absenteeism time in hours'].sum()

absen = [drink_sum, smok_sum, drink_smok_sum, abs_sum]
pie_labels = ['drinker', 'smoker', 'drinker & smoker', 'No drinker/smoker']
fig1, ax1 = plt.subplots(figsize=(5, 5))
fig1.subplots_adjust(0.3, 0, 1, 1)
 
theme = plt.get_cmap('Blues')
ax1.set_prop_cycle("color", [theme(1. * i / len(absen))
                             for i in range(len(absen))])
 
_, _ = ax1.pie(absen, startangle=90, radius=1800)
 
ax1.axis('equal')

total = sum(absen)
plt.legend(
    loc='upper left',
    labels=['%s, %1.1f%%' % (
        l, (float(s) / total) * 100)
            for l, s in zip(pie_labels, absen)],
    prop={'size': 11},
    bbox_to_anchor=(0.0, 1),
    bbox_transform=fig1.transFigure
)

ax1.set_title('Absenteeism by Social Drinkers/Smokers')
plt.show()

#Abesenteeism by day of the week
plt.figure(figsize=(12,6))
sns.lmplot(x='Age',y='Absenteeism time in hours',data=df,hue='Day of the week',height=5,aspect=2)
roa = df.groupby('Reason for absence')
roa['Absenteeism time in hours'].max()
# Reason 6,9,11,12,13,19 are highest reasons for absenteeism
sns.countplot(x='Absenteeism time in hours',data=df)
#Let's see how other columns are correlated to target variable Absenteeism hours in a heat map. 
def correlation_heatmap(li):
    correlations = li.corr()

    fig, ax = plt.subplots(figsize=(9,9))
    sns.heatmap(correlations, vmax=1.0, center=0, fmt='.2f',cmap='Blues',
                square=True, linewidths=.5, annot=True, cbar_kws={"shrink": .70})
    plt.show();
    
correlation_heatmap(df[['Age','Education','Seasons','Transportation expense' ,'Month of absence','Distance from Residence to Work','Service time','Work load Average/day ','Body mass index','Hit target','Absenteeism time in hours']])
#box plot for absenteeism hours to show outliers
df.boxplot('Absenteeism time in hours')<jupyter_output><empty_output><jupyter_text>Regression models<jupyter_code>#creating a function for printing r-2 score and RMSE
from sklearn.metrics import r2_score, mean_squared_error as MSE
import warnings
warnings.filterwarnings('ignore')
def R2ScoreRMSE(y_test,y_pred):
    test_score = r2_score(y_test, y_pred)
    rmse_test = MSE(y_test, y_pred) ** (1/2)
    print(f'Test R-2 score: {test_score:>5.3}')
    print(f'Test RMSE     : {rmse_test:.4f}')

#Since the problem statement asks to predict number of absenteeism hours, trying out regression model first
#We will build model using RandomForestRegressor and LinearRegression
#First we are using RandomForestRegressor
from sklearn.ensemble import RandomForestRegressor
from sklearn.preprocessing import OneHotEncoder
from sklearn import preprocessing
#following features are dropped:
#  1) ID    
#  2) Weight  --> BMI covers this feature
#  3) Height  --> BMI covers this feature
df_features = df[['Reason for absence', 'Month of absence', 'Day of the week', 'Seasons', 
                  'Transportation expense','Distance from Residence to Work', 'Service time', 
                  'Age', 'Work load Average/day ', 'Hit target','Disciplinary failure', 'Education', 
                  'Son', 'Social drinker', 'Social smoker', 'Pet', 'Body mass index' ]]
X = df_features
y = df['Absenteeism time in hours']
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.3, random_state=0)
model = RandomForestRegressor(n_estimators=1000, random_state=0, oob_score=True, n_jobs=-1)
model.fit(X_train,y_train) 
y_pred = model.predict(X_test)

#Accuracy score evaluation
R2ScoreRMSE(y_test,y_pred)
# RMSE score is too high and also R-2 score is very low.
#Let's check the importance of features in the model
ax = (pd.Series(model.feature_importances_, index=X.columns)
   .nlargest(19)
   .plot(kind='barh', figsize=(8,6), color='lightblue'))
plt.show()
# LinearRegression Moddel
from sklearn.linear_model import LinearRegression
from sklearn.preprocessing import StandardScaler

X = df_features
y = df['Absenteeism time in hours']
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.3, random_state=0)
model =LinearRegression()
a = model.fit(X_train, y_train)
y_pred = model.predict(X_test)

#Accuracy score evaluation
R2ScoreRMSE(y_test,y_pred)
# LinearRegression Moddel
# if we exclude outlier absenteeism hours, we are able to improve the scores
# we have encoded categorical values using get_dummies. Total features increased to 75 because of this.
# We have scaled all the columns using StandardScaler
from sklearn.linear_model import LinearRegression
from sklearn.preprocessing import StandardScaler

# exclude rows having outliers
df_excluding_outliers = df_original[df_original['Absenteeism time in hours']<=24]

df_features_excluding_outliers = df_excluding_outliers[['Reason for absence', 'Month of absence', 'Day of the week', 'Seasons', 
                  'Transportation expense','Distance from Residence to Work', 'Service time', 
                  'Age', 'Work load Average/day ', 'Hit target','Disciplinary failure', 'Education', 
                  'Son', 'Social drinker', 'Social smoker', 'Pet', 'Body mass index' ]]

# encode categorical attributes
cols_to_transform = ['Reason for absence','Month of absence', 'Day of the week', 'Seasons','Education','Son','Pet' ]
df_features_excluding_outliers = pd.get_dummies(df_features_excluding_outliers, columns = cols_to_transform )

# scale all the attributes
X = df_features_excluding_outliers
autoscaler = StandardScaler()
X_scaled = autoscaler.fit_transform(X)


y = df_excluding_outliers['Absenteeism time in hours']
X_train, X_test, y_train, y_test = train_test_split(X_scaled, y, test_size=0.3, random_state=0)
model =LinearRegression()
a = model.fit(X_train, y_train)
y_pred = model.predict(X_test)

#Accuracy score evaluation
R2ScoreRMSE(y_test,y_pred)
df_features_excluding_outliers.info()
# Linear Regression Model with Feature reduction
# skipping some of the features by using Recursive Feature Elimination (RFE) 
# We tried with various number of features and found that with 38 features we get 
# best balance between good R-2 score with less number of features.

from sklearn.feature_selection import RFE
from sklearn.linear_model import LogisticRegression
from sklearn.linear_model import LinearRegression

X = df_features_excluding_outliers
autoscaler = StandardScaler()
X_scaled = autoscaler.fit_transform(X)
y = df_excluding_outliers['Absenteeism time in hours']
X_train, X_test, y_train, y_test = train_test_split(X_scaled, y, test_size=0.3, random_state=0)

model = LinearRegression()
model = RFE(model, 38)
fit = model.fit(X_train, y_train)
print("Num Features: %s" % (fit.n_features_))
print("Selected Features: %s" % (fit.support_))
print("Feature Ranking: %s" % (fit.ranking_))
y_pred = model.predict(X_test)

#Accuracy score evaluation
R2ScoreRMSE(y_test,y_pred)<jupyter_output>Num Features: 38
Selected Features: [False False False False False False  True False False False  True False
 False False False False False False False False False False False False
 False False False False False False False False False False False False
 False False  True  True  True  True  True  True  True  True  True  True
  True  True False  True  True  True  True  True  True  True  True  True
  True  True  True  True  True  True  True  True  True  True  True  True
  True  True  True]
Feature Ranking: [30 33 32 31 35 38  1 36 34 37  1 15 29 27 26 22 12 20 11 28 19  7 25 16
 10 21  8 24 18 13  9 17  3 23  6 14  5  4  1  1  1  1  1  1  1  1  1  1
  1  1  2  1  1  1  1  1  1  1  1  1  1  1  1  1  1  1  1  1  1  1  1  1
  1  1  1]
Test R-2 score: 0.127
Test RMSE     : 3.9322<jupyter_text>Classification Models<jupyter_code>#Since we are not able to improve sores using Regression models,
#Let's convert the target variable to categorical variable to predict in classification model
#using RandomForestClassifier
from sklearn.ensemble import RandomForestClassifier
from sklearn.metrics import classification_report

# Creating 4 absenteeism hour classes:
# absent_class = 0: absent for upto 4 hours
# absent_class = 1: absent for more then 4 hours not exceeding 8 hours
# absent_class = 2: absent for more then 8 hours not exceeding 24 hours
# absent_class = 3: absent for more then 24 hours
df_features = df[['Reason for absence', 'Month of absence', 'Day of the week', 'Seasons', 
                  'Transportation expense','Distance from Residence to Work', 'Service time', 
                  'Age', 'Work load Average/day ', 'Hit target','Disciplinary failure', 'Education', 
                  'Son', 'Social drinker', 'Social smoker', 'Pet', 'Body mass index' ]]

aba=[]
for i in df['Absenteeism time in hours']:
    if i <= 4:
        aba.append(0)
    elif i <= 8:
        aba.append(1)
    elif i<=24:
        aba.append(2)
    else:
        aba.append(3)
df['absent_class']=aba

X = df_features
y = df['absent_class']
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.3, random_state=42)

model = RandomForestClassifier(n_estimators=100, random_state=0, oob_score=True, n_jobs=-1)
model.fit(X_train,y_train)
y_pred = model.predict(X_test)

print("Accuracy:",metrics.accuracy_score(y_test, y_pred))
print(classification_report(y_test, y_pred))
#Contributions of features in the above model
ax = (pd.Series(model.feature_importances_, index=X.columns)
   .nlargest(19)
   .plot(kind='barh', figsize=(8,6), color='orange'))
plt.title('Contributions of Features')
plt.show()
#Using  Naive Bayes model
from sklearn.naive_bayes import GaussianNB
#Create a Gaussian Classifier
gnb = GaussianNB()

#Train the model using the training sets
gnb.fit(X_train, y_train)

#Predict the response for test dataset
y_pred = gnb.predict(X_test)
print("Accuracy:",metrics.accuracy_score(y_test, y_pred))
print(classification_report(y_test, y_pred))
#Using Decision Tree classifier
from sklearn.tree import DecisionTreeClassifier

clf_gini = DecisionTreeClassifier(criterion = "gini", random_state = 100,
                               max_depth=3, min_samples_leaf=5)
clf_gini.fit(X_train, y_train)

y_pred = clf_gini.predict(X_test)
print("Accuracy:",metrics.accuracy_score(y_test, y_pred))
print(classification_report(y_test, y_pred))<jupyter_output>precision    recall  f1-score   support

           0       0.87      0.82      0.85       130
           1       0.65      0.82      0.72        78
           2       0.00      0.00      0.00         9
           3       0.00      0.00      0.00         5

   micro avg       0.77      0.77      0.77       222
   macro avg       0.38      0.41      0.39       222
weighted avg       0.74      0.77      0.75       222
{'fraction_non_alphanumeric': 0.11977563444832068, 'fraction_numerical': 0.026677611327724195, 'mean_word_length': 4.633911368015414, 'pattern_counts': {'":': 1, '<': 23, '<?xml version=': 0, '>': 22, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 8, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '5', 'dir': 'jupyter-structured-clean-dedup', 'id': '468422', 'license': 'BSD-3-Clause', 'n_tokens_mistral': 5250, 'n_tokens_neox': 4867, 'n_words': 1479, 'path': 'Absenteeism_at_work.ipynb', 'repo_name': 'alkemoha2/Absenteeism_at_work', 'size': '14619'}