<jupyter_start><jupyter_text>Importing Packages<jupyter_code>import matplotlib.pyplot as plt #For plotting
import numpy as np #Provides fast numerical computing such as arrays and linear algebra
import pandas as pd #Provides R like data structures and a high level API to work with data
from pandas import DataFrame, Series
import seaborn as sns #To make your plots look better
import apyori as ap
import pickle
from apyori import apriori #Apriori Algorithm
import mlxtend as ml
from mlxtend.frequent_patterns import apriori
from mlxtend.frequent_patterns import association_rules
from mlxtend.preprocessing import TransactionEncoder<jupyter_output><empty_output><jupyter_text>Loading data in dataframe<jupyter_code>Attritiondata = pd.read_csv("/Users/yashpasar/Desktop/MSIM/707/HW_1/employee_attrition.csv")
pd.set_option('display.max_columns', 100)<jupyter_output><empty_output><jupyter_text>Number of entries<jupyter_code>print("Number of entries: " + str(len(Attritiondata.index)))
Attritiondata.head()<jupyter_output><empty_output><jupyter_text>Exploratory Data Analysis<jupyter_code>Attritiondata.info()<jupyter_output><class 'pandas.core.frame.DataFrame'>
RangeIndex: 1176 entries, 0 to 1175
Data columns (total 35 columns):
Age                         1176 non-null int64
Attrition                   1176 non-null object
BusinessTravel              1176 non-null object
DailyRate                   1176 non-null int64
Department                  1176 non-null object
DistanceFromHome            1174 non-null float64
Education                   1176 non-null int64
EducationField              1176 non-null object
EmployeeCount               1176 non-null int64
EmployeeNumber              1176 non-null int64
EnvironmentSatisfaction     1176 non-null int64
Gender                      1175 non-null object
HourlyRate                  1176 non-null int64
JobInvolvement              1176 non-null int64
JobLevel                    1175 non-null float64
JobRole                     1176 non-null object
JobSatisfaction             1176 non-null int64
MaritalStatus               1176 non-null object
MonthlyIncome     [...]<jupyter_text>Dropping Null Values<jupyter_code>Refined_data = Attritiondata.drop(['EmployeeCount', 'EmployeeNumber', 'MonthlyRate', 'StandardHours'], axis=1)
def replacenull(col):
    array = Refined_data[Refined_data[col] != np.nan][col]
    Refined_data[col] = Refined_data[col].replace(np.NaN, array.median())
    return Refined_data[col]

def replacenullmode(col):
    array = Refined_data[Refined_data[col] != np.nan][col]
    Refined_data[col] = Refined_data[col].replace(np.NaN, array.mode()[0])
    return Refined_data[col] 

replacenull('DistanceFromHome')
replacenull('JobLevel')
replacenull('PercentSalaryHike')
replacenull('PerformanceRating')
replacenull('RelationshipSatisfaction')
replacenull('TotalWorkingYears')
replacenull('YearsSinceLastPromotion')
replacenullmode('Gender')
replacenullmode('OverTime')
Refined_data.isnull().sum()
str(len(Refined_data.index))
Refined_data.columns
Attrition = Refined_data.groupby('Attrition')
Attrition.mean()
Refined_data.hist(figsize=(20,20))
plt.show()
Refined_data.describe()
# Find correlations with the target and sort
Attritiondata['Target'] = Attritiondata['Attrition'].apply(
    lambda x: 0 if x == 'No' else 1)
correlations = Attritiondata.corr()['Target'].sort_values()
print('Most Positive Correlations: \n', correlations.tail(5))
print('\nMost Negative Correlations: \n', correlations.head(5))
# Calculate correlations
corr = Attritiondata.corr()
mask = np.zeros_like(corr)
mask[np.triu_indices_from(mask)] = True
# Heatmap
plt.figure(figsize=(15, 10))
sns.heatmap(corr,
            vmax=.5,
            mask=mask,
            # annot=True, fmt='.2f',
            linewidths=.2, cmap="YlGnBu")
Refined_data['YearsWithCurrManager'].value_counts()<jupyter_output><empty_output><jupyter_text>Discretizing all variables<jupyter_code>Refined_data['Age'] = pd.qcut(Refined_data.Age, 3, labels = ['young_age','med_age','old_age'])
Refined_data["Education"] = pd.cut(Refined_data.Education, [0,3,5], labels = ['low_edu', 'high_edu'])
Refined_data['DailyRate'] = pd.cut(Refined_data.DailyRate, [0,461,796,1162,1500], labels = ['Q1_Daily_rate', 'Q2_Daily_rate', 'Q3_Daily_rate', 'Q4_Daily_rate'])
Refined_data['DistanceFromHome'] = pd.cut(Refined_data.DistanceFromHome, [0,2,7,14,224], labels = ['Q1_Dist', 'Q2_Dist', 'Q3_Dist', 'Q4_Dist'])
Refined_data['EnvironmentSatisfaction'] = pd.cut(Refined_data.EnvironmentSatisfaction, [0,2,3,4,4.1], labels = ['Q1_Env', 'Q2_Env', 'Q3_Env', 'Q4_Env'])
Refined_data['HourlyRate'] = pd.qcut(Refined_data.HourlyRate, 4, labels = ['Q1_HourlyRate', 'Q2_HourlyRate', 'Q3_HourlyRate', 'Q4_HourlyRate'])
Refined_data['JobInvolvement'] = pd.cut(Refined_data.JobInvolvement, [1,2,3,4], labels = ['Q1_JobInvolvement', 'Q2_JobInvolvement', 'Q3_JobInvolvement'])
Refined_data["JobLevel"] = pd.cut(Refined_data.JobLevel, [0,3,5], labels = ['low_JobLevel', 'high_JobLevel'])
Refined_data["JobSatisfaction"] = pd.cut(Refined_data.JobSatisfaction, [0,3,4], labels = ['low_JobSatisfaction', 'high_JobSatisfaction'])
Refined_data['MonthlyIncome'] = pd.qcut(Refined_data.MonthlyIncome, 4, labels = ['Q1_MonthlyIncome', 'Q2_MonthlyIncome', 'Q3_MonthlyIncome', 'Q4_MonthlyIncome'])
Refined_data['NumCompaniesWorked'] = pd.qcut(Refined_data.NumCompaniesWorked, 4, labels = ['Q1_NumCompaniesWorked', 'Q2_NumCompaniesWorked', 'Q3_NumCompaniesWorked', 'Q4_NumCompaniesWorked'])
Refined_data['PercentSalaryHike'] = pd.qcut(Refined_data.PercentSalaryHike, 4, labels = ['Q1_PercentSalaryHike', 'Q2_PercentSalaryHike', 'Q3_PercentSalaryHike', 'Q4_PercentSalaryHike'])
Refined_data['PerformanceRating'] = pd.cut(Refined_data.PerformanceRating, [0,3,5], labels = ['Three', 'Four'])
Refined_data['RelationshipSatisfaction'] = pd.cut(Refined_data.RelationshipSatisfaction, [0,2,4], labels = ['low_Satisfaction', 'high_Satisfaction'])
Refined_data['StockOptionLevel'] = pd.cut(Refined_data.StockOptionLevel, [-1,0,1,2,3], labels = ['Level0', 'Level1', 'Leve2', 'Level3'])
Refined_data['TotalWorkingYears'] = pd.qcut(Refined_data.TotalWorkingYears, 4, labels = ['Q1_TotalWorkingYears', 'Q2_TotalWorkingYears', 'Q3_TotalWorkingYears', 'Q4_TotalWorkingYears'])
Refined_data['TrainingTimesLastYear'] = pd.cut(Refined_data.TrainingTimesLastYear, [0,2,4,6], labels = ['0-2 years', '2-4 years', '4-6 years'])
Refined_data['WorkLifeBalance'] = pd.cut(Refined_data.WorkLifeBalance, [0,2,4], labels = ['Low_WorkLifeBalance', 'High_WorkLifeBalance'])
Refined_data['YearsAtCompany'] = pd.qcut(Refined_data.YearsAtCompany, 4, labels = ['Q1_YearsAtCompany', 'Q2_YearsAtCompany', 'Q3_YearsAtCompany', 'Q4_YearsAtCompany'])
Refined_data['YearsInCurrentRole'] = pd.cut(Refined_data.YearsInCurrentRole, [0,4.5,9,13.5,18], labels = ['0-4.5 years', '4.5-9 years', '9-13.5 years', '13.5-18 years'])
Refined_data['YearsSinceLastPromotion'] = pd.cut(Refined_data.YearsSinceLastPromotion, [-1,5,10,15], labels = ['0-5 years', '5-10 years', '10-15 years'])
Refined_data['YearsWithCurrManager'] = pd.cut(Refined_data.YearsWithCurrManager, [-1,5,10,15,219], labels = ['0-5 years', '5-10 years', '10-15 years', '15-21 years'])
Refined_data.head()
Refined_data = Refined_data.astype('str')
Refined_data[Refined_data.columns] = Refined_data.apply(lambda x: x.str.strip())
Refined_data = Refined_data.astype('object')
Refined_data.head()
Refined_data1 = pd.DataFrame({col:str(col)+'=' for col in Refined_data}, index=Refined_data.index) + Refined_data.astype(str)
Refined_data1.head()<jupyter_output><empty_output><jupyter_text>Converting Wide format to Long format<jupyter_code>melted_data = pd.melt(Refined_data1)
melted_data.head()
frequency = melted_data.groupby(by=['value'])['value'].count().sort_values(ascending=True)
freq_itemset = pd.DataFrame({'item':frequency.index, 'frequency':frequency.values})
freq_itemset.tail(10)
g = sns.barplot(data=freq_itemset.tail(10), x='item', y='frequency')
g.set_xticklabels(g.get_xticklabels(), rotation=90)
plt.show()<jupyter_output><empty_output><jupyter_text>Creating a list in a list<jupyter_code>records = []
for i in range(0,len(Refined_data1)):
    records.append([str(Refined_data1.values[i,j]) 
    for j in range(0, len(Refined_data1.columns))])
    
record=pickle.dump(records, open( "records.pkl", "wb" ) )
frequent_itemset = ap.apriori(records, min_support=0.5, min_confidence=0.5,
                              min_lift=1,min_length=2)
results = list(frequent_itemset)
len(results)
results[1:5]
te = TransactionEncoder()
te_ary = te.fit(records).transform(records)
df = pd.DataFrame(te_ary, columns=te.columns_)
df.head()
frequent_itemsets = apriori(df,min_support = 0.5, use_colnames=True)
frequent_itemsets['length'] = frequent_itemsets['itemsets'].apply(lambda x: len(x))
frequent_itemsets.sort_values(by='support',ascending=False).head(10)<jupyter_output><empty_output><jupyter_text>Generating rules through default settings<jupyter_code>rules = association_rules(frequent_itemsets, metric="lift", min_threshold=1)
rules[(rules['lift']>1) & (rules['confidence'] > 0.5)].head(10)
b = sns.scatterplot(data=rules,x='support',y='confidence',hue='lift')
plt.show()<jupyter_output><empty_output><jupyter_text>Generating rules when min_support is 0.78<jupyter_code>frequent_itemsets1 = apriori(df,min_support = 0.78, use_colnames=True)
frequent_itemsets1['length'] = frequent_itemsets1['itemsets'].apply(lambda x: len(x))
frequent_itemsets1.sort_values(by='support',ascending=False).head(10)
rules1 = association_rules(frequent_itemsets1, metric="lift", min_threshold=1)
rules1.head()
rules1[(rules1['lift']>1) & (rules1['confidence'] > 0.8)].head(10)
b = sns.scatterplot(data=rules1,x='support',y='confidence',hue='lift')
plt.show()<jupyter_output><empty_output><jupyter_text>Rules to predict Attrition<jupyter_code>def SupervisedApriori(data,consequent,min_supp,min_conf,min_lift):
    frequent_itemsets = apriori(data, min_supp, use_colnames=True)
    rules = association_rules(frequent_itemsets, metric="confidence", min_threshold=min_conf)
    rules = rules[rules['lift']>min_lift]
    sup_rules = pd.DataFrame()
    for i in consequent:
        df = rules[rules['consequents'] == {i}]
        sup_rules = sup_rules.append(df,ignore_index = True)
    return(sup_rules)
    
sup_rules = SupervisedApriori(df,consequent = ['Attrition=No','Attrition=Yes'],min_supp=0.6, min_conf=0.6, min_lift=1)
sup_rules.sort_values(by='support',ascending=False).head(5)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.14152469782050062, 'fraction_numerical': 0.03511944418007043, 'mean_word_length': 6.681286549707602, 'pattern_counts': {'":': 0, '<': 45, '<?xml version=': 0, '>': 50, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '11', 'dir': 'jupyter-structured-clean-dedup', 'id': '190423', 'license': 'MIT', 'n_tokens_mistral': 3984, 'n_tokens_neox': 3714, 'n_words': 602, 'path': 'HW1.ipynb', 'repo_name': 'yashpasar/Association-Rule-Mining', 'size': '10507'}