<jupyter_start><jupyter_text>Salary Prediction Project EDApass<jupyter_code>__author__ = '<NAME>'
__email__ = '<EMAIL>'
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
import seaborn as sns
%matplotlib inline
# Read in files and print the information
train_feature_df = pd.read_csv('../data/raw/train_features.csv')
train_target_df = pd.read_csv('../data/raw/train_salaries.csv')
test_feature_df = pd.read_csv('../data/raw/test_features.csv')
train_feature_df.head()
train_target_df.head()
test_feature_df.head()
train_feature_df.info()
train_target_df.info()
test_feature_df.info()
train_feature_df.duplicated().sum()
train_target_df.duplicated().sum()
test_feature_df.duplicated().sum()
train_feature_df.isnull().sum()
test_feature_df.isnull().sum()
train_target_df.isnull().sum()
numeric_cols = [col for col in train_feature_df.columns if train_feature_df[col].dtype == 'int64']
numeric_cols
categorical_cols = [col for col in train_feature_df.columns if train_feature_df[col].dtype == 'object']
categorical_cols
train_feature_df.describe().transpose()
train_feature_df.describe(include='O').transpose()<jupyter_output><empty_output><jupyter_text>For the `categorical` features, we can see that there's `63` Company the most frequent Company is `COMP39`, `8` types of job, most frequent is `SENIRO` level, `5` typs of education degree, the most frequent is `HIGH_SCHOOL`, and `9` types of degree major, the most frequent is `NONE` which mean maybe the individual forgot what major they are studied, but we assume is the individual not willing to provide. What's more, there are `7` types of Industry, the most frequent is `WEB` industry.<jupyter_code># Merge the features and salaries on jobId, delete original file to save memory
train_df = pd.merge(train_feature_df, train_target_df, on='jobId')
del train_feature_df
del train_target_df
train_df.head(10)
train_df.info()
plt.figure(figsize = (14, 6))
plt.subplot(1,2,1)
sns.boxplot(train_df.salary)
plt.subplot(1,2,2)
sns.distplot(train_df.salary, bins=20)
plt.show()
stat = train_df.salary.describe()
print(stat, '\n')
IQR = stat['75%'] - stat['25%']
upper = stat['75%'] + 1.5 * IQR
lower = stat['25%'] - 1.5 * IQR
print(f'The upper and lower bounds for suspected outliers are {upper} and {lower}.')
#check potential outlier below lower bound
train_df[train_df.salary < 8.5]
#check potential outlier above upper bound
train_df.loc[train_df.salary > 220.5, 'jobType'].value_counts()
# Check most suspicious potential outliers above upper bound
train_df[(train_df.salary > 220.5) & (train_df.jobType == 'JUNIOR')]<jupyter_output><empty_output><jupyter_text>These entries with zero salary do not appear to be volunteer positions. We are confident that they are instances of missing/corrupt data and should be removed from the training set.The high-salary potential outliers all appear to be legitimate data. Most roles are C-level executive roles and the junior positions are in industries that are well known for high salaries (oil, finance). We determine these entries to be legitimate and will not remove them.<jupyter_code># Remove data with zero salaries
train_df = train_df[train_df.salary > 8.5]
def plot_feature(df, col):
    '''
    Make plot for each features
    left, the distribution of samples on the feature
    right, the dependance of salary on the feature
    '''
    plt.figure(figsize = (14, 6))
    plt.subplot(1, 2, 1)
    if df[col].dtype == 'int64':
        df[col].value_counts().sort_index().plot()
    else:
        #change the categorical variable to category type and order their level by the mean salary
        #in each category
        mean = df.groupby(col)['salary'].mean()
        df[col] = df[col].astype('category')
        levels = mean.sort_values().index.tolist()
        df[col].cat.reorder_categories(levels, inplace=True)
        df[col].value_counts().plot()
    plt.xticks(rotation=45)
    plt.xlabel(col)
    plt.ylabel('Counts')
    plt.subplot(1, 2, 2)

    if df[col].dtype == 'int64' or col == 'companyId':
        #plot the mean salary for each category and fill between the (mean - std, mean + std)
        mean = df.groupby(col)['salary'].mean()
        std = df.groupby(col)['salary'].std()
        mean.plot()
        plt.fill_between(range(len(std.index)), mean.values-std.values, mean.values + std.values, \
                         alpha = 0.1)
    else:
        sns.boxplot(x = col, y = 'salary', data=df)
    
    plt.xticks(rotation=45)
    plt.ylabel('Salaries')
    plt.show()
plot_feature(train_df, 'companyId')
plot_feature(train_df, 'jobType')
plot_feature(train_df, 'degree')
plot_feature(train_df, 'major')
plot_feature(train_df, 'industry')
plot_feature(train_df, 'yearsExperience')
plot_feature(train_df, 'milesFromMetropolis')
def encode_label(df, col):
    #encode the categories using average salary for each category to replace label
    cat_dict ={}
    cats = df[col].cat.categories.tolist()
    for cat in cats:
        cat_dict[cat] = train_df[train_df[col] == cat]['salary'].mean()   
    df[col] = df[col].map(cat_dict)
for col in train_df.columns:
    if train_df[col].dtype.name == "category":
        encode_label(train_df, col)
# Correlations between selected features and response
# jobId is discarded because it is unique for individual
fig = plt.figure(figsize=(12, 10))
features = ['companyId', 'jobType', 'degree', 'major', 'industry', 'yearsExperience', 'milesFromMetropolis']
sns.heatmap(train_df[features + ['salary']].corr(), cmap='Blues', annot=True)
plt.xticks(rotation=45)
plt.show()<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.12444046553267682, 'fraction_numerical': 0.01289167412712623, 'mean_word_length': 4.764705882352941, 'pattern_counts': {'":': 1, '<': 16, '<?xml version=': 0, '>': 18, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '3', 'dir': 'jupyter-structured-clean-dedup', 'id': '496826', 'license': 'MIT', 'n_tokens_mistral': 1891, 'n_tokens_neox': 1819, 'n_words': 512, 'path': 'notebooks/EDA.ipynb', 'repo_name': 'JL1829/EmployeeSalaryPrediction', 'size': '5595'}