<jupyter_start><jupyter_text>We convert all outlier data into "nan" instead, in the next step all "nan" data will be converted into mean or median. Here we only analyze the outlier data from train dataset. Because we want to analyze based on the class. While the test dataset has no label/class.<jupyter_code>df_train = pd.read_csv('./3.num_train.csv')
print(df_train.shape)
by_class = df_train.groupby('Best Performance')
df_train_positive = by_class.get_group(1)
df_train_negative = by_class.get_group(0)
print(df_train_positive.shape)
print(df_train_negative.shape)
def calculateIQRvalue(x) :
    Q3, Q1 = np.nanpercentile(x, [75 ,25])
    IQR = Q3 - Q1
    lower_range = Q1-1.5*IQR
    upper_range = Q3+1.5*IQR
    return(Q1,Q3,IQR,lower_range,upper_range)<jupyter_output><empty_output><jupyter_text>job_duration_in_current_job_level<jupyter_code>sns.boxplot(x='Best Performance', y='job_duration_in_current_job_level', data=df_train)
_,_,_,lower_pos, upper_pos = calculateIQRvalue(df_train_positive['job_duration_in_current_job_level'])
_,_,_,lower_neg, upper_neg = calculateIQRvalue(df_train_negative['job_duration_in_current_job_level'])
print(lower_pos, upper_pos)
print(lower_neg, upper_neg)
df_train.loc[(df_train['job_duration_in_current_job_level'] > upper_pos) & (df_train['Best Performance'] == 1), 'job_duration_in_current_job_level'] = nan
df_train.loc[(df_train['job_duration_in_current_job_level'] < lower_pos) & (df_train['Best Performance'] == 1), 'job_duration_in_current_job_level'] = nan
df_train.loc[(df_train['job_duration_in_current_job_level'] > upper_neg) & (df_train['Best Performance'] == 0), 'job_duration_in_current_job_level'] = nan
df_train.loc[(df_train['job_duration_in_current_job_level'] < lower_neg) & (df_train['Best Performance'] == 0), 'job_duration_in_current_job_level'] = nan
sns.boxplot(x='Best Performance', y='job_duration_in_current_job_level', data=df_train)<jupyter_output><empty_output><jupyter_text>job_duration_in_current_person_level<jupyter_code>sns.boxplot(x='Best Performance', y='job_duration_in_current_person_level', data=df_train)
_,_,_,lower_pos, upper_pos = calculateIQRvalue(df_train_positive['job_duration_in_current_person_level'])
_,_,_,lower_neg, upper_neg = calculateIQRvalue(df_train_negative['job_duration_in_current_person_level'])
print(lower_pos, upper_pos)
print(lower_neg, upper_neg)
df_train.loc[(df_train['job_duration_in_current_person_level'] > upper_pos) & (df_train['Best Performance'] == 1), 'job_duration_in_current_person_level'] = nan
df_train.loc[(df_train['job_duration_in_current_person_level'] < lower_pos) & (df_train['Best Performance'] == 1), 'job_duration_in_current_person_level'] = nan
df_train.loc[(df_train['job_duration_in_current_person_level'] > upper_neg) & (df_train['Best Performance'] == 0), 'job_duration_in_current_person_level'] = nan
df_train.loc[(df_train['job_duration_in_current_person_level'] < lower_neg) & (df_train['Best Performance'] == 0), 'job_duration_in_current_person_level'] = nan
sns.boxplot(x='Best Performance', y='job_duration_in_current_person_level', data=df_train)<jupyter_output><empty_output><jupyter_text>job_duration_in_current_branch<jupyter_code>sns.boxplot(x='Best Performance', y='job_duration_in_current_branch', data=df_train)
_,_,_,lower_pos, upper_pos = calculateIQRvalue(df_train_positive['job_duration_in_current_branch'])
_,_,_,lower_neg, upper_neg = calculateIQRvalue(df_train_negative['job_duration_in_current_branch'])
print(lower_pos, upper_pos)
print(lower_neg, upper_neg)
df_train.loc[(df_train['job_duration_in_current_branch'] > upper_pos) & (df_train['Best Performance'] == 1), 'job_duration_in_current_branch'] = nan
df_train.loc[(df_train['job_duration_in_current_branch'] < lower_pos) & (df_train['Best Performance'] == 1), 'job_duration_in_current_branch'] = nan
df_train.loc[(df_train['job_duration_in_current_branch'] > upper_neg) & (df_train['Best Performance'] == 0), 'job_duration_in_current_branch'] = nan
df_train.loc[(df_train['job_duration_in_current_branch'] < lower_neg) & (df_train['Best Performance'] == 0), 'job_duration_in_current_branch'] = nan
sns.boxplot(x='Best Performance', y='job_duration_in_current_branch', data=df_train)<jupyter_output><empty_output><jupyter_text>number_of_dependences<jupyter_code>sns.boxplot(x='Best Performance', y='number_of_dependences', data=df_train)
_,_,_,lower_pos, upper_pos = calculateIQRvalue(df_train_positive['number_of_dependences'])
_,_,_,lower_neg, upper_neg = calculateIQRvalue(df_train_negative['number_of_dependences'])
print(lower_pos, upper_pos)
print(lower_neg, upper_neg)
df_train.loc[(df_train['number_of_dependences'] > upper_pos) & (df_train['Best Performance'] == 1), 'number_of_dependences'] = nan
df_train.loc[(df_train['number_of_dependences'] < lower_pos) & (df_train['Best Performance'] == 1), 'number_of_dependences'] = nan
df_train.loc[(df_train['number_of_dependences'] > upper_neg) & (df_train['Best Performance'] == 0), 'number_of_dependences'] = nan
df_train.loc[(df_train['number_of_dependences'] < lower_neg) & (df_train['Best Performance'] == 0), 'number_of_dependences'] = nan
sns.boxplot(x='Best Performance', y='number_of_dependences', data=df_train)<jupyter_output><empty_output><jupyter_text>GPA<jupyter_code>sns.boxplot(x='Best Performance', y='GPA', data=df_train)
_,_,_,lower_pos, upper_pos = calculateIQRvalue(df_train_positive['GPA'])
_,_,_,lower_neg, upper_neg = calculateIQRvalue(df_train_negative['GPA'])
print(lower_pos, upper_pos)
print(lower_neg, upper_neg)
df_train.loc[(df_train['GPA'] > upper_pos) & (df_train['Best Performance'] == 1), 'GPA'] = nan
df_train.loc[(df_train['GPA'] < lower_pos) & (df_train['Best Performance'] == 1), 'GPA'] = nan
df_train.loc[(df_train['GPA'] > upper_neg) & (df_train['Best Performance'] == 0), 'GPA'] = nan
df_train.loc[(df_train['GPA'] < lower_neg) & (df_train['Best Performance'] == 0), 'GPA'] = nan
sns.boxplot(x='Best Performance', y='GPA', data=df_train)<jupyter_output><empty_output><jupyter_text>job_duration_from_training<jupyter_code>sns.boxplot(x='Best Performance', y='job_duration_from_training', data=df_train)
_,_,_,lower_pos, upper_pos = calculateIQRvalue(df_train_positive['job_duration_from_training'])
_,_,_,lower_neg, upper_neg = calculateIQRvalue(df_train_negative['job_duration_from_training'])
print(lower_pos, upper_pos)
print(lower_neg, upper_neg)
df_train.loc[(df_train['job_duration_from_training'] > upper_pos) & (df_train['Best Performance'] == 1), 'job_duration_from_training'] = nan
df_train.loc[(df_train['job_duration_from_training'] < lower_pos) & (df_train['Best Performance'] == 1), 'job_duration_from_training'] = nan
df_train.loc[(df_train['job_duration_from_training'] > upper_neg) & (df_train['Best Performance'] == 0), 'job_duration_from_training'] = nan
df_train.loc[(df_train['job_duration_from_training'] < lower_neg) & (df_train['Best Performance'] == 0), 'job_duration_from_training'] = nan
sns.boxplot(x='Best Performance', y='job_duration_from_training', data=df_train)<jupyter_output><empty_output><jupyter_text>branch_rotation<jupyter_code>sns.boxplot(x='Best Performance', y='branch_rotation', data=df_train)
_,_,_,lower_pos, upper_pos = calculateIQRvalue(df_train_positive['branch_rotation'])
_,_,_,lower_neg, upper_neg = calculateIQRvalue(df_train_negative['branch_rotation'])
print(lower_pos, upper_pos)
print(lower_neg, upper_neg)
df_train.loc[(df_train['branch_rotation'] > upper_pos) & (df_train['Best Performance'] == 1), 'branch_rotation'] = nan
df_train.loc[(df_train['branch_rotation'] < lower_pos) & (df_train['Best Performance'] == 1), 'branch_rotation'] = nan
df_train.loc[(df_train['branch_rotation'] > upper_neg) & (df_train['Best Performance'] == 0), 'branch_rotation'] = nan
df_train.loc[(df_train['branch_rotation'] < lower_neg) & (df_train['Best Performance'] == 0), 'branch_rotation'] = nan
sns.boxplot(x='Best Performance', y='branch_rotation', data=df_train)<jupyter_output><empty_output><jupyter_text>job_rotation<jupyter_code>sns.boxplot(x='Best Performance', y='job_rotation', data=df_train)
_,_,_,lower_pos, upper_pos = calculateIQRvalue(df_train_positive['job_rotation'])
_,_,_,lower_neg, upper_neg = calculateIQRvalue(df_train_negative['job_rotation'])
print(lower_pos, upper_pos)
print(lower_neg, upper_neg)
df_train.loc[(df_train['job_rotation'] > upper_pos) & (df_train['Best Performance'] == 1), 'job_rotation'] = nan
df_train.loc[(df_train['job_rotation'] < lower_pos) & (df_train['Best Performance'] == 1), 'job_rotation'] = nan
df_train.loc[(df_train['job_rotation'] > upper_neg) & (df_train['Best Performance'] == 0), 'job_rotation'] = nan
df_train.loc[(df_train['job_rotation'] < lower_neg) & (df_train['Best Performance'] == 0), 'job_rotation'] = nan
sns.boxplot(x='Best Performance', y='job_rotation', data=df_train)<jupyter_output><empty_output><jupyter_text>assign_of_otherposition<jupyter_code>sns.boxplot(x='Best Performance', y='assign_of_otherposition', data=df_train)
_,_,_,lower_pos, upper_pos = calculateIQRvalue(df_train_positive['assign_of_otherposition'])
_,_,_,lower_neg, upper_neg = calculateIQRvalue(df_train_negative['assign_of_otherposition'])
print(lower_pos, upper_pos)
print(lower_neg, upper_neg)
df_train.loc[(df_train['assign_of_otherposition'] > upper_pos) & (df_train['Best Performance'] == 1), 'assign_of_otherposition'] = nan
df_train.loc[(df_train['assign_of_otherposition'] < lower_pos) & (df_train['Best Performance'] == 1), 'assign_of_otherposition'] = nan
df_train.loc[(df_train['assign_of_otherposition'] > upper_neg) & (df_train['Best Performance'] == 0), 'assign_of_otherposition'] = nan
df_train.loc[(df_train['assign_of_otherposition'] < lower_neg) & (df_train['Best Performance'] == 0), 'assign_of_otherposition'] = nan
sns.boxplot(x='Best Performance', y='assign_of_otherposition', data=df_train)<jupyter_output><empty_output><jupyter_text>annual leave<jupyter_code>sns.boxplot(x='Best Performance', y='annual leave', data=df_train)
_,_,_,lower_pos, upper_pos = calculateIQRvalue(df_train_positive['annual leave'])
_,_,_,lower_neg, upper_neg = calculateIQRvalue(df_train_negative['annual leave'])
print(lower_pos, upper_pos)
print(lower_neg, upper_neg)
df_train.loc[(df_train['annual leave'] > upper_pos) & (df_train['Best Performance'] == 1), 'annual leave'] = nan
df_train.loc[(df_train['annual leave'] < lower_pos) & (df_train['Best Performance'] == 1), 'annual leave'] = nan
df_train.loc[(df_train['annual leave'] > upper_neg) & (df_train['Best Performance'] == 0), 'annual leave'] = nan
df_train.loc[(df_train['annual leave'] < lower_neg) & (df_train['Best Performance'] == 0), 'annual leave'] = nan
sns.boxplot(x='Best Performance', y='annual leave', data=df_train)<jupyter_output><empty_output><jupyter_text>sick_leaves<jupyter_code>sns.boxplot(x='Best Performance', y='sick_leaves', data=df_train)
_,_,_,lower_pos, upper_pos = calculateIQRvalue(df_train_positive['sick_leaves'])
_,_,_,lower_neg, upper_neg = calculateIQRvalue(df_train_negative['sick_leaves'])
print(lower_pos, upper_pos)
print(lower_neg, upper_neg)
df_train.loc[(df_train['sick_leaves'] > upper_pos) & (df_train['Best Performance'] == 1), 'sick_leaves'] = nan
df_train.loc[(df_train['sick_leaves'] < lower_pos) & (df_train['Best Performance'] == 1), 'sick_leaves'] = nan
df_train.loc[(df_train['sick_leaves'] > upper_neg) & (df_train['Best Performance'] == 0), 'sick_leaves'] = nan
df_train.loc[(df_train['sick_leaves'] < lower_neg) & (df_train['Best Performance'] == 0), 'sick_leaves'] = nan
sns.boxplot(x='Best Performance', y='sick_leaves', data=df_train)<jupyter_output><empty_output><jupyter_text>Last_achievement_%<jupyter_code>sns.boxplot(x='Best Performance', y='Last_achievement_%', data=df_train)
_,_,_,lower_pos, upper_pos = calculateIQRvalue(df_train_positive['Last_achievement_%'])
_,_,_,lower_neg, upper_neg = calculateIQRvalue(df_train_negative['Last_achievement_%'])
print(lower_pos, upper_pos)
print(lower_neg, upper_neg)
df_train.loc[(df_train['Last_achievement_%'] > upper_pos) & (df_train['Best Performance'] == 1), 'Last_achievement_%'] = nan
df_train.loc[(df_train['Last_achievement_%'] < lower_pos) & (df_train['Best Performance'] == 1), 'Last_achievement_%'] = nan
df_train.loc[(df_train['Last_achievement_%'] > upper_neg) & (df_train['Best Performance'] == 0), 'Last_achievement_%'] = nan
df_train.loc[(df_train['Last_achievement_%'] < lower_neg) & (df_train['Best Performance'] == 0), 'Last_achievement_%'] = nan
sns.boxplot(x='Best Performance', y='Last_achievement_%', data=df_train)<jupyter_output><empty_output><jupyter_text>Achievement_above_100%_during3quartal<jupyter_code>sns.boxplot(x='Best Performance', y='Achievement_above_100%_during3quartal', data=df_train)
_,_,_,lower_pos, upper_pos = calculateIQRvalue(df_train_positive['Achievement_above_100%_during3quartal'])
_,_,_,lower_neg, upper_neg = calculateIQRvalue(df_train_negative['Achievement_above_100%_during3quartal'])
print(lower_pos, upper_pos)
print(lower_neg, upper_neg)
df_train.loc[(df_train['Achievement_above_100%_during3quartal'] > upper_pos) & (df_train['Best Performance'] == 1), 'Achievement_above_100%_during3quartal'] = nan
df_train.loc[(df_train['Achievement_above_100%_during3quartal'] < lower_pos) & (df_train['Best Performance'] == 1), 'Achievement_above_100%_during3quartal'] = nan
df_train.loc[(df_train['Achievement_above_100%_during3quartal'] > upper_neg) & (df_train['Best Performance'] == 0), 'Achievement_above_100%_during3quartal'] = nan
df_train.loc[(df_train['Achievement_above_100%_during3quartal'] < lower_neg) & (df_train['Best Performance'] == 0), 'Achievement_above_100%_during3quartal'] = nan
sns.boxplot(x='Best Performance', y='Achievement_above_100%_during3quartal', data=df_train)
df_train.to_csv("4.train_without_outlier.csv", index=False)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.15161617625742863, 'fraction_numerical': 0.00898680968256269, 'mean_word_length': 12.268269230769231, 'pattern_counts': {'":': 0, '<': 83, '<?xml version=': 0, '>': 83, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '14', 'dir': 'jupyter-structured-clean-dedup', 'id': '273914', 'license': 'MIT', 'n_tokens_mistral': 5132, 'n_tokens_neox': 4892, 'n_words': 674, 'path': '4.1. Dealing with outlier data (train data only).ipynb', 'repo_name': 'vhiefa/People-Analytics-BRI-Challenge-EDA', 'size': '13798'}