<jupyter_start><jupyter_text>0. Initial EDA on the dataset<jupyter_code># Check the shape of data

num_rows = df.shape[0]
num_cols = df.shape[1]

print(f"# of rows: {num_rows}, # of cols: {num_cols}")
# Check the columns and their dtypes of data

df.info(verbose=True)
# Check the proportion of NULL values in each column

df.isnull().mean()
# Check some interesting columns in detail
 
df['Professional'].value_counts()
df['MajorUndergrad'].value_counts()
df['Salary'].hist()
df['JobSatisfaction'].hist()<jupyter_output><empty_output><jupyter_text>1. About Programming Language<jupyter_code># Get a subtable with selected columns and row only for Professional developer

target_columns = ['HaveWorkedLanguage', 'JobSatisfaction', 'Salary']
lang_df = df[df['Professional'] == 'Professional developer'][target_columns]
lang_df.head()
lang_df['HaveWorkedLanguage'].value_counts()
# Compute the number of programming language appears in the survey

languages = pd.Series(itertools.chain(*[x.split('; ') for x in lang_df['HaveWorkedLanguage'] if isinstance(x, str)]))
language_counts = languages.value_counts()
language_counts
pd.DataFrame(language_counts, columns=['Count'])
language_rank = language_counts.rank(ascending=False)
language_rank
plt.figure(figsize=(10,10))
language_counts.plot(kind='bar');
plt.title('Popular Language');
plt.xlabel('Programming Language');
plt.ylabel('Frequency');<jupyter_output><empty_output><jupyter_text>For frequency, top 5 languages are Javascript, SQL, Java, C and Python<jupyter_code>lang_df_satisf = lang_df.dropna(subset=['JobSatisfaction'])
lang_df_satisf
# Compute the average job satisfaction for each programming language

expand_df = pd.DataFrame(columns = ['JobSatisfaction', 'Language'])

for idx, row in lang_df_satisf.iterrows():
    job = row['JobSatisfaction']
    if isinstance(row['HaveWorkedLanguage'], str):
        lang = row['HaveWorkedLanguage'].split('; ')
        for l in lang:
            temp = {
                    'JobSatisfaction': [job], 
                    'Language': [l]}
            expand_df = expand_df.append(pd.DataFrame(temp))
job_mean = expand_df.groupby('Language').mean()['JobSatisfaction'].sort_values(ascending=False)
job_rank = job_mean.rank(ascending=False)
plt.figure(figsize=(10,10))
job_mean.plot(kind='bar');
plt.title('Average Job Satisfaction per Language');
plt.xlabel('Programming Language');
plt.ylabel('Job Satisfaction');<jupyter_output><empty_output><jupyter_text>For Job satisfaction, top 5 languages are SmallTalk. Objective-C, Elixir, CoffeeScript, TypeScript    Python is located in 11th<jupyter_code>lang_df_salary = lang_df.dropna(subset=['Salary'])
lang_df_salary
# Compute the average salary for each programming language

salary_df = pd.DataFrame(columns = ['Language', 'Salary'])

for idx, row in lang_df_salary.iterrows():
    salary = row['Salary']
    if isinstance(row['HaveWorkedLanguage'], str):
        lang = row['HaveWorkedLanguage'].split('; ')
        for l in lang:
            temp = {'Salary': [salary], 
                    'Language': [l]}
            salary_df = salary_df.append(pd.DataFrame(temp))
salary_mean = salary_df.groupby('Language').mean()['Salary'].sort_values(ascending=False)
salary_rank = salary_mean.rank(ascending=False)
plt.figure(figsize=(10,10))
salary_mean.plot(kind='bar');
plt.title('Average Salary per Language');
plt.xlabel('Programming Language');
plt.ylabel('Salary');<jupyter_output><empty_output><jupyter_text>For Salary, top 5 languages are Clojure, Smalltalk, Rust, Go and Hack    Python is located in the middle<jupyter_code>avg_ranking_reciproacal = (3 / (job_rank + language_rank + salary_rank)).sort_values()
avg_ranking_reciproacal
plt.figure(figsize=(10,10))
avg_ranking_reciproacal.plot(kind='barh');
plt.title('Average Ranking Reciprocal of Programming Language');
plt.xlabel('Average Ranking Reciprocal');
plt.ylabel('Programming Language');<jupyter_output><empty_output><jupyter_text>About Company<jupyter_code>target = ['HomeRemote', 'CompanyType', 'CompanySize', 'JobSatisfaction']
comp_df = df.dropna(subset=['JobSatisfaction'])[target]
comp_df.head()
def get_avg_satisfaction(df, target):
    """ Function to get average jobsatisfaction score per target column"""
    
    temp_df = df[[target, 'JobSatisfaction']]
    return temp_df.groupby(target).mean()['JobSatisfaction'].sort_values(ascending=False)
target = 'HomeRemote'
get_avg_satisfaction(comp_df, target)
target = 'CompanySize'
get_avg_satisfaction(comp_df, target)
target = 'CompanyType'
get_avg_satisfaction(comp_df, target)<jupyter_output><empty_output><jupyter_text>3. What features determine high job satisfaction / Salary?    Using all features, find features that have an impact on the target (job satisfaction / salary)    To utilize machine learning model, we need to preprocess the data    Prerpocess steps:        1. Missing Value            Delete                Delete columns with more than 80% of null values                Delete rows with more than 80% of null values            Impute                For categorical data, impute 'Null' in Null values                For numerical data, impute mean of each column in NUll values        2. Ordinary Encoding            Convert categorical data into ordinal data (numerical data)        3. Modeling            Make a feature selection model   3.1 Preprocess  3.1.1 Delete missing values<jupyter_code># Check the proportaion of Null values for each column

new_df = df.dropna(subset=['Salary'], axis=0)
new_df.isnull().mean()
def get_null_row(df, thres=0.9) -> list:
    """ This returns index of rows with a proportion of null values above the threshold """
    return df.index[df.isnull().mean(axis=1) > thres].tolist()

def get_null_col(df, thres=0.9) -> list:
    """ This returns column name with a proportion of null values above the threshold """
    temp_df = df.isnull().mean()
    return temp_df[temp_df > thres].index.tolist()
new_df = new_df.drop(get_null_col(new_df, 0.8), axis=1)
new_df = new_df.drop(get_null_row(new_df, 0.8), axis=0)
new_df.reset_index(drop=True, inplace=True)<jupyter_output><empty_output><jupyter_text>3.1.2 Impute Row/Column missing values<jupyter_code># Impute a value for columns that have a relatively small percentage of Null values

new_df.isnull().mean()
# For columns with continuous values, impute mean of the column
con_df = new_df.select_dtypes(include=['int', 'float'])
fill_null = lambda col: col.fillna(col.mean())
con_df = con_df.apply(fill_null, axis=0)
con_df
# For columns with categorical values, impute string Null into the column

cat_df = new_df.select_dtypes(include=['object'])
fill_null = lambda col: col.fillna('Null')
cat_df = cat_df.apply(fill_null, axis=0)
cat_df<jupyter_output><empty_output><jupyter_text>3.1.3 Convert Category column to Ordinal column<jupyter_code>oe = OrdinalEncoder()
oe.fit(cat_df)
cat_df_encoding = oe.transform(cat_df)
cat_df_encoding = pd.DataFrame(cat_df_encoding, columns=cat_df.columns)
cat_df_encoding<jupyter_output><empty_output><jupyter_text>3.1.4 Merge Con_df with Cat_df<jupyter_code>new_df = pd.concat([con_df, cat_df_encoding], axis=1)
new_df<jupyter_output><empty_output><jupyter_text>3.2 Build feature selection model<jupyter_code>def get_feature_score(df, target):
    """ This returns feature selection scores for the target column """
    features = [x for x in df.columns if x != target]

    x_data = df[features]
    y_data = df[target]

    fs = SelectKBest(score_func=mutual_info_regression, k='all')
    fs.fit(x_data, y_data)

    fs_score = {}
    for i, s in enumerate(fs.scores_):
        fs_score[x_data.columns[i]] = s
    fs_score = {k:v for k,v in sorted(fs_score.items(), key=lambda item: item[1], reverse=True)}
    
    return fs_score<jupyter_output><empty_output><jupyter_text>3.2.1 Model for Salary<jupyter_code>target = 'Salary'
fs_score = get_feature_score(new_df, target)
fs_score
plt.figure(figsize=(15,50))
plt.grid(True, axis='x', color='gray', alpha=0.5, linestyle='--')
plt.barh(*zip(*fs_score.items()))
plt.gca().invert_yaxis()
plt.title('Feature Importance for Salary')
plt.show()<jupyter_output><empty_output><jupyter_text>3.2.2 Model For JobSatisfaction<jupyter_code>target = 'JobSatisfaction'
fs_score = get_feature_score(new_df, target)
fs_score
plt.figure(figsize=(15,50))
plt.grid(True, axis='x', color='gray', alpha=0.5, linestyle='--')
plt.barh(*zip(*fs_score.items()))
plt.gca().invert_yaxis()
plt.title(f'Feature Importance for {target}')
plt.show()<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1258659152283668, 'fraction_numerical': 0.009862627685804861, 'mean_word_length': 5.522205206738132, 'pattern_counts': {'":': 0, '<': 53, '<?xml version=': 0, '>': 57, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '13', 'dir': 'jupyter-structured-clean-dedup', 'id': '238536', 'license': 'MIT', 'n_tokens_mistral': 2908, 'n_tokens_neox': 2786, 'n_words': 657, 'path': 'developer_survey_analysis.ipynb', 'repo_name': 'sucpark/stackoverflow_survey_analysis', 'size': '8517'}