<jupyter_start><jupyter_text>Job title and seniority<jupyter_code>def seniority(title):
    if 'sr' in title.lower() or 'senior' in title.lower() or 'sr' in title.lower() or 'lead' in title.lower() or 'principal' in title.lower():
            return 'senior'
    elif 'jr' in title.lower() or 'jr.' in title.lower():
        return 'jr'
    else:
        return 'na'
df['job_simp'] = df['Job Title'].apply(title_simplifier)
df.job_simp.value_counts()
df['seniority'] = df['Job Title'].apply(seniority)
df.seniority.value_counts()<jupyter_output><empty_output><jupyter_text>Fix state Los Angeles<jupyter_code>df['job_state']= df.job_state.apply(lambda x: x.strip() if x.strip().lower() != 'los angeles' else 'CA')
df.job_state.value_counts()<jupyter_output><empty_output><jupyter_text>Job description length<jupyter_code>df['desc_len'] = df['Job Description'].apply(lambda x: len(x))
df['desc_len']<jupyter_output><empty_output><jupyter_text>Competitor count<jupyter_code>df['num_comp'] = df['Competitors'].apply(lambda x: len(x.split(',')) if x != '-1' else 0)
df['num_comp']
df.columns<jupyter_output><empty_output><jupyter_text>Hourly wage to annual<jupyter_code>df['min_salary'] = df.apply(lambda x: x.min_salary*2 if x.hourly ==1 else x.min_salary, axis =1)
df['max_salary'] = df.apply(lambda x: x.max_salary*2 if x.hourly ==1 else x.max_salary, axis =1)
df[df.hourly==1][['hourly', 'min_salary', 'max_salary']]<jupyter_output><empty_output><jupyter_text>Remove new line from job title<jupyter_code>df['company_txt'] = df.company_txt.apply(lambda x: x.replace('\n',''))
df['company_txt']
df.describe()
df.columns
df.Rating.hist()
df.avg_salary.hist()
df.desc_len.hist()
df.boxplot(column = ['age','avg_salary','Rating'])
df.boxplot(column = 'Rating')<jupyter_output><empty_output><jupyter_text>Co-Relationsolder companies has much more competition as seen in 5th and 4th column<jupyter_code>cmap = sns.diverging_palette(360, 10, as_cmap=True)
sns.heatmap(df[['age','avg_salary','Rating','desc_len','num_comp']].corr(),vmax=.25, center=0, cmap=cmap,
            square=True, linewidths=.5, cbar_kws={"shrink": .5})
df.columns
df_cat = df[['Size','Type of ownership', 'Sector', 'Revenue', 'job_state','same_state', 'python_yn', 'R_yn',
       'spark', 'aws', 'excel', 'job_simp', 'seniority']]
for i in df_cat.columns:
    cat_num = df_cat[i].value_counts()
    print("graph for %s: total = %d" % (i, len(cat_num)))
    chart = sns.barplot(x=cat_num.index, y=cat_num)
    chart.set_xticklabels(chart.get_xticklabels(), rotation=90)
    plt.show()<jupyter_output>graph for Size: total = 9<jupyter_text>The companies with bigger data will take only 20 inputs<jupyter_code>df_big_cat = df[['Location','Headquarters','company_txt', 'Industry']]
for i in df_big_cat[['Location','Headquarters','company_txt', 'Industry']].columns:
    cat_num = df_big_cat[i].value_counts()[:20]
    print("graph for %s: total = %d" % (i, len(cat_num)))
    chart = sns.barplot(x=cat_num.index, y=cat_num)
    chart.set_xticklabels(chart.get_xticklabels(), rotation=90)
    plt.show()
df.columns
pd.pivot_table(df, index = 'job_simp', values = 'avg_salary')
pd.pivot_table(df, index = ['job_simp','seniority'], values = 'avg_salary')
pd.pivot_table(df, index = 'job_state', values = 'avg_salary').sort_values('avg_salary', ascending = False)
pd.options.display.max_rows
pd.set_option('display.max_rows', None)
pd.pivot_table(df, index = ['job_state','job_simp'], values = 'avg_salary').sort_values('job_state', ascending = True)<jupyter_output><empty_output><jupyter_text>aggfunc count will give you the count of jobs inside avg_salary<jupyter_code>pd.pivot_table(df, index = ['job_state','job_simp'], values = 'avg_salary', aggfunc = 'count').sort_values('job_state', ascending = True)<jupyter_output><empty_output><jupyter_text>only sort data scientist vs job state<jupyter_code>pd.pivot_table(df[df.job_simp == 'data scientist'], index = 'job_state', values = 'avg_salary').sort_values('avg_salary', ascending = False)
df.columns
df_pivots = df[['Rating', 'Industry', 'Sector', 'Revenue', 'num_comp', 'hourly', 'employer_provided', 'python_yn', 'R_yn', 'spark', 'aws', 'excel', 'Type of ownership','avg_salary']]
for i in df_pivots.columns:
    print(i)
    print(pd.pivot_table(df_pivots,index =i, values = 'avg_salary').sort_values('avg_salary', ascending = False))
pd.pivot_table(df_pivots, index = 'Revenue', columns = 'python_yn', values = 'avg_salary', aggfunc = 'count')<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.17420360882156383, 'fraction_numerical': 0.006683002895967922, 'mean_word_length': 7.017857142857143, 'pattern_counts': {'":': 1, '<': 40, '<?xml version=': 0, '>': 40, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '10', 'dir': 'jupyter-structured-clean-dedup', 'id': '474018', 'license': 'MIT', 'n_tokens_mistral': 1755, 'n_tokens_neox': 1708, 'n_words': 312, 'path': '.ipynb_checkpoints/data_eda-checkpoint.ipynb', 'repo_name': 'shhubhxm/data-science-salary-prediction', 'size': '4489'}