<jupyter_start><jupyter_text>Import Libraries<jupyter_code>import pandas as pd
from sklearn.preprocessing import LabelEncoder
from sklearn.impute import SimpleImputer
import seaborn as sns
sns.set()
import matplotlib.pyplot as plt
from sklearn.cluster import KMeans,DBSCAN
from sklearn.neighbors import NearestNeighbors
from sklearn.preprocessing import StandardScaler
import numpy as np
from sklearn import metrics
from sklearn.decomposition import PCA
from sklearn.model_selection import GridSearchCV, KFold, train_test_split
import xgboost as xgb
from scipy.stats import uniform, randint<jupyter_output><empty_output><jupyter_text>Read Data<jupyter_code>data = pd.read_excel(r"D:/python projects/read_files/dataset.xlsx")<jupyter_output><empty_output><jupyter_text>General Column Handling<jupyter_code>data.columns = data.columns.str.replace(' ','')
data.columns = data.columns.str.lower()<jupyter_output><empty_output><jupyter_text>Specifies features and targetI will specify the features which are the mental health issues category, also the target which is iamunemployed which i will rename as employed to easier the calling<jupyter_code>data = data[['lackofconcentration', 'anxiety', 'depression', 'obsessivethinking',
        'moodswings', 'panicattacks', 'compulsivebehavior', 'tiredness', 'age',
        'gender', 'householdincome','iamunemployed']]
data.rename(columns = {'iamunemployed' : 'unemployed'},inplace=True)
data.head(5)
data.shape<jupyter_output><empty_output><jupyter_text>Null CheckingI will check missing values and if they exist i will impute them with median if their dtypes is numerical or  most frequent if their dtype is numerical<jupyter_code>data.isnull().sum()
def fillna(df):
    for column in df.columns:
        if df[column].dtype == object:
            simo = SimpleImputer(strategy='most_frequent')
            df[column] = simo.fit_transform(df[column].values.reshape(-1,1))
        else:
            sime = SimpleImputer(strategy='median')
            df[column] = sime.fit_transform(df[column].values.reshape(-1,1))
            
fillna(data)<jupyter_output><empty_output><jupyter_text>Checking duplicated recordsI will check for duplications<jupyter_code>data.duplicated().sum()
data.drop_duplicates(keep=False,inplace=True)
data.info()
data.describe()
plt.figure(figsize=(12,8))
sns.heatmap(data.corr(),annot=True,annot_kws={'size':12},cmap="BuGn")
plt.tick_params(labelsize=12)<jupyter_output><empty_output><jupyter_text>Specifying top 5th features and targetI will specify top 5th features and target<jupyter_code>data = data[['panicattacks','compulsivebehavior',
             'depression', 'anxiety',
             'tiredness','unemployed'
             ]]<jupyter_output><empty_output><jupyter_text>Create dataframe for classification laterI will make copy of data before i scale it so i don't need to work twice on preparing again. This will be used for the final result conlusion and classification<jupyter_code>cat_data = data.copy()
final_data = data.copy()<jupyter_output><empty_output><jupyter_text>Cluster Model Building  PCA to optimize DBSCAN<jupyter_code>scp = StandardScaler()
data = scp.fit_transform(data)

pca = PCA(n_components=1)
pca = pca.fit(data)
pca_data = pca.transform(data)<jupyter_output><empty_output><jupyter_text>DBSCAN PCA Fitting<jupyter_code>dbscan = DBSCAN(eps=0.3, min_samples = np.log(data.shape[0]))
dbc_pca = dbscan.fit_predict(pca_data)<jupyter_output><empty_output><jupyter_text>DBSCAN Base Model<jupyter_code>dbc = dbscan.fit_predict(data)<jupyter_output><empty_output><jupyter_text>Cluster EvaluationI will compare how good PCA outperform the base model without PCA<jupyter_code>def validate(X,model):
    s_metrics = metrics.silhouette_score(X, model)
    dbi_metrics = metrics.davies_bouldin_score(X, model)
    print('Silhouette score : %s'%s_metrics)
    print('DBI score : %s'%dbi_metrics)
validate(data,dbc)
validate(pca_data,dbc_pca)<jupyter_output>Silhouette score : 0.750866715857714
DBI score : 0.3059602855691396<jupyter_text>Classification Model Building<jupyter_code>cat_data['cluster'] = dbc_pca
cat_data.columns
cat_data = pd.get_dummies(cat_data,columns=['panicattacks', 'compulsivebehavior', 'depression', 'anxiety',
       'tiredness'])<jupyter_output><empty_output><jupyter_text>Plot CorrelationsBefore i really build the model i'd like to see how related my features towards my target, which is not that bad<jupyter_code>corrdf = pd.get_dummies(cat_data).corr()
plt.figure(figsize=(12,8))
sns.heatmap(pd.get_dummies(cat_data).corr(),annot=True)
X = np.array(cat_data.drop(['unemployed'],1))
y = np.array(cat_data['unemployed'])

X = StandardScaler().fit_transform(X)

X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2)<jupyter_output><empty_output><jupyter_text>XGBClassifier Hyperparameter Tuning with GridSearchCV + Fitting<jupyter_code>xgb_model = xgb.XGBClassifier()
params = {
        'min_child_weight': [1, 5, 10],
        'gamma': [5, 10, 20 ,30],
        'subsample': [0.6, 0.8, 1.0],
        'colsample_bytree': [0.6, 0.8, 1.0],
        'max_depth': [10, 20, 50],
        'learning_rate': [0.01,0.02,0.1,0.3]
}

grid = GridSearchCV(estimator= xgb_model,param_grid =params,
                    refit=True,verbose=1,cv =10,n_jobs=-1)
grid.fit(X_train,y_train)<jupyter_output>Fitting 10 folds for each of 1296 candidates, totalling 12960 fits<jupyter_text>XGBClassifier Base Model<jupyter_code>xgb_base_model = xgb.XGBClassifier()
xgb_base_model.fit(X_train,y_train)<jupyter_output><empty_output><jupyter_text>Classification Model EvaluationI will compare the performance of my regularized XGBClassifier model with base XGBClassifier model<jupyter_code>def evaluate(model,X_train,y_train,X_test,y_test):
    y_pred = model.predict(X_test)
    accuracy = metrics.accuracy_score(y_test, y_pred)
    print('Training set score : %s'%model.score(X_train,y_train))
    print('Test set score : %s'%model.score(X_test,y_test))
    print('Accuracy score : %s'%accuracy)
best_param = grid.best_estimator_
evaluate(best_param,X_train,y_train,X_test,y_test)
print(best_param)
evaluate(xgb_base_model,X_train,y_train,X_test,y_test)<jupyter_output>Training set score : 0.6729559748427673
Test set score : 0.65
Accuracy score : 0.65<jupyter_text>Conclusion  1. Plot Unemployed Based On Clusters<jupyter_code>y_pred = best_param.predict(X_test)
unemployed_new = np.append(y_train,y_pred)
final_data = final_data.assign(unemployed=unemployed_new)
final_data['cluster'] = dbc_pca
final_data['unemployed'] = final_data['unemployed'].replace(to_replace=[1.0,0.0],value=['yes','no'])<jupyter_output><empty_output><jupyter_text>Plotting counts of each clusterThis shows that top 3 clusters with most numbers are cluster 2,4,5<jupyter_code>plt.figure(figsize=(10,6))
sns.countplot(x = final_data['cluster'],data = final_data)
plt.title('Counts of each cluster')<jupyter_output><empty_output><jupyter_text>Plotting Unemployed yes vs Unemployed no in each clusterThis shows the top 3 clusters with the most unemployed status, which are cluster 2,5,4 and that is shows that the numbers of unemployed and the employed has significant different. I could see that most people are employed in every clusters so this implies that mental health issues don't stop someone to be unemployed<jupyter_code>plt.figure(figsize=(10,6))
sns.set(font_scale=1.2)
sns.countplot(x="cluster", hue="unemployed", data=final_data)
plt.title('Unemployed yes vs Unemployed no in each cluster')<jupyter_output><empty_output><jupyter_text>Plotting Unemployed yes vs Unemployed no in generalFrom the plot i could learn that people with mental health issues still have high chances to be employed so it does not effect employment much by still having 71,36% to be employed which means again,having mental health issues don't stop someone to be unemployed<jupyter_code>plt.figure(figsize=(7,7))
sns.set(font_scale=1.2)
plt.pie(final_data.groupby(['unemployed'])['unemployed'].count(),
        autopct=lambda p : '{:.2f}%  ({:,.0f})'.format(p,(p/100)*final_data['unemployed'].count()))
plt.legend(final_data['unemployed'].unique(),loc='center right',
            bbox_to_anchor=(1, 0, 0.5, 1))
plt.title('Unemployed yes vs Unemployed no in general')<jupyter_output><empty_output><jupyter_text>Plotting Combinations of mental health issues in each clusterFrom my clusters of people who are unemployed it shows the combination of the mental health issues, it seems cluster 2 and 6 don't have any mental heath issues at all. And people in cluster 4 and cluster 3 seems to have all mental health issues significancy.<jupyter_code>unemployed = final_data.iloc[:,np.r_[0:4,-2:0]][final_data['unemployed']=='yes']
unemployed = unemployed.groupby(['cluster'])['panicattacks', 'compulsivebehavior', 'depression', 'anxiety'].sum()
unemployed.plot.bar(figsize=(10,8))
plt.title('Combinations of mental health issues in each cluster')<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.11290322580645161, 'fraction_numerical': 0.02060931899641577, 'mean_word_length': 6.717372515125324, 'pattern_counts': {'":': 0, '<': 86, '<?xml version=': 0, '>': 86, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '22', 'dir': 'jupyter-structured-clean-dedup', 'id': '645222', 'license': 'MIT', 'n_tokens_mistral': 3195, 'n_tokens_neox': 2989, 'n_words': 680, 'path': 'ClusteringClassificationUnemploymentMentalHealthIssues.ipynb', 'repo_name': 'rmd1999/personal_project', 'size': '8928'}