<jupyter_start><jupyter_text>My Questions    - What's the relationship between age started coding and salary    - What's the relationship between years coded and salary    - What's the relationship between years coded professionally and salary  Gather<jupyter_code>#import libraries
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
from scipy import stats
import seaborn as sns

%matplotlib inline

df = pd.read_csv('./2020_survey_results_public.csv')
schema = pd.read_csv('./2020_survey_results_schema.csv')


#taking a look at the data to understand it's structure
df.head()
#using the schema file to understand all the different categories in the data
schema.head()<jupyter_output><empty_output><jupyter_text>Assess<jupyter_code>#First, I want to get an understanding of how many different ages we're dealing with here and where most of the responses are
Age1stCode = df.Age1stCode.value_counts()
Age1stCode.head()
#And I want to take a look at how many values there are total
Age1stCode.shape
#A bar chart to show the distribution
(Age1stCode[:20]).plot(kind="bar");
plt.title("Age of response (top 20)");
#Same process for the YearsCode column (for the years the respondent has been coding for)
YearsCode = df.YearsCode.value_counts()
YearsCode.head()
YearsCode.shape
(YearsCode[:20]).plot(kind="bar");
plt.title("Years Coded (top 20)");
#repeating the process for YearsCodePro (number of years the respondent has coded professionally)
YearsCodePro = df.YearsCodePro.value_counts()
YearsCodePro.head()
YearsCodePro.shape
(YearsCodePro[:20]).plot(kind="bar");
plt.title("Years Coded Professionally (top 20)");<jupyter_output><empty_output><jupyter_text>Clean  Age1stCode<jupyter_code>#first removing all the files with no value for the age they first coded or their salary converted to USD
df_no_na_Age1st = df.dropna(subset=['Age1stCode', 'ConvertedComp'], axis=0)
df_no_na_Age1st.head()
#We won't be able to make a scatter plot unless all the values are numerical
#So I'll change the values with letters in them into numbers
df_no_na_Age1st.loc[(df.Age1stCode == 'Younger than 5 years'),'Age1stCode'] = 4 
df_no_na_Age1st.loc[(df.Age1stCode == 'Older than 85'),'Age1stCode'] = 86
#Let's check what kind of type this column is
print(df_no_na_Age1st['Age1stCode'].dtypes)
#object, not what we want
#I'll convert it to numeric
df_no_na_Age1st['Age1stCode'] = df_no_na_Age1st['Age1stCode'].apply(pd.to_numeric)
#check to make sure it worked
print(df_no_na_Age1st['Age1stCode'].dtypes)
#bingo
#now group the averages by the age the respondent first coded
avgAge1st = df_no_na_Age1st.groupby('Age1stCode', as_index=False).mean()
avgAge1st
# the values for 85 and 84 are clear outliers. Let's graph it for illustration.

avgAge1st.plot(x='Age1stCode', y='ConvertedComp', s=10, kind="scatter");
plt.title("Age / Salary")
#84 and 85 throw off the whole graph, how many values are there for those ages?
over_85 = df_no_na_Age1st[df_no_na_Age1st["Age1stCode"] >= 84]
over_85
# 44 and 45 are also outliers let's take a look at those values
forty_four = df_no_na_Age1st[df_no_na_Age1st["Age1stCode"] == 44]
forty_five = df_no_na_Age1st[df_no_na_Age1st["Age1stCode"] == 45]
forty_four
forty_five
#so it's pretty clear those making over a million dollars a year salary are throwing the numbers off.
#we'll remove the troubesome items in 44 and 45 and dropping 85 and 86 entirely
df_no_na_Age1st = df_no_na_Age1st.drop([1581,47807])
avgAge1st = df_no_na_Age1st.groupby('Age1stCode', as_index=False).mean()
avg_clean = avgAge1st.drop([47,48])
#making sure 85 and 86 are removed
avg_clean.tail()
# a quick scatter plot to see the results
avg_clean.plot(x='Age1stCode', y='ConvertedComp', s=30, kind="scatter");
plt.title("Age / Salary")<jupyter_output><empty_output><jupyter_text>YearsCode<jupyter_code>#first removing all the files with no value for the the years they've been coding or their salary converted to USD
df_no_na_YearsCode = df.dropna(subset=['YearsCode', 'ConvertedComp'], axis=0)
df_no_na_YearsCode.head()
#same process for years code, change the string values into numeric values
df_no_na_YearsCode.loc[(df.YearsCode == 'Less than 1 year'),'YearsCode'] = 0.1
df_no_na_YearsCode.loc[(df.YearsCode == 'More than 50 years'),'YearsCode'] = 51
df_no_na_YearsCode['YearsCode'] = df_no_na_YearsCode['YearsCode'].apply(pd.to_numeric)
avgYearsCode = df_no_na_YearsCode.groupby('YearsCode', as_index=False).mean()
avgYearsCode.head()
avgYearsCode.plot(x='YearsCode', y='ConvertedComp', s=30, kind="scatter");
plt.title("Years Coded / Salary")<jupyter_output><empty_output><jupyter_text>YearsCodePro<jupyter_code>#first removing all the files with no value for the years they've been coding professionally or their salary in USD
df_no_na_YearsPro = df.dropna(subset=['YearsCodePro', 'ConvertedComp'], axis=0)
df_no_na_YearsPro.head()
#same process for yearscodepro
df_no_na_YearsPro.loc[(df.YearsCodePro == 'Less than 1 year'),'YearsCodePro'] = 0.1
df_no_na_YearsPro.loc[(df.YearsCodePro == 'More than 50 years'),'YearsCodePro'] = 51
df_no_na_YearsPro['YearsCodePro'] = df_no_na_YearsPro['YearsCodePro'].apply(pd.to_numeric)
avgYearsPro = df_no_na_YearsPro.groupby('YearsCodePro', as_index=False).mean()
avgYearsPro.head()
avgYearsPro.plot(x='YearsCodePro', y='ConvertedComp', s=30, kind="scatter");
plt.title("Years Coded Professionally / Salary")<jupyter_output><empty_output><jupyter_text>Analyze<jupyter_code>#getting the basic description of the data for the age the respondent first coded
avg_clean.describe()
#the basic description for the data of the amount of years the respondent coded
avgYearsCode.describe()
#looking at the description of the the average years the respondent coded professionally
avgYearsPro.describe()
#full breakdown for the averages by the age the respondent first started coding
avg_clean
#breakdown of the ful data for the averages by the years the respondent had been coding
avgYearsCode
#breakdown of the ful data for the averages by the years the respondent had been coding professionally
avgYearsPro<jupyter_output><empty_output><jupyter_text>Model<jupyter_code># getting the values for the line of best fit for the age first coded scatter plot
slope, intercept, r_value, p_value, std_err = stats.linregress(avg_clean['Age1stCode'],avg_clean['ConvertedComp'])

print("Equation: y={0:.1f}x+{1:.1f}".format(slope,intercept))
print('Correlation Coefficien: r=',r_value, sep='')
# getting the values for the line of best fit for the total years coded scatter plot
slope, intercept, r_value, p_value, std_err = stats.linregress(avgYearsCode['YearsCode'],avgYearsCode['ConvertedComp'])

print("Equation: y={0:.1f}x+{1:.1f}".format(slope,intercept))
print('Correlation Coefficien: r=',r_value, sep='')
# getting the values for the line of best fit for the total years coded professionally scatter plot
slope, intercept, r_value, p_value, std_err = stats.linregress(avgYearsPro['YearsCodePro'],avgYearsPro['ConvertedComp'])

print("Equation: y={0:.1f}x+{1:.1f}".format(slope,intercept))
print('Correlation Coefficien: r=',r_value, sep='')<jupyter_output>Equation: y=2546.1x+84248.3
Correlation Coefficien: r=0.571425948087<jupyter_text>Visualize<jupyter_code># Scatter plot with line of best fit for the age the respondent started coding
Age1stCodePlot = sns.regplot(avg_clean['Age1stCode'],avg_clean['ConvertedComp'])
plt.xlabel("Age Respondent Started Coding")
plt.ylabel("Salary (USD)")
# Scatter plot with line of best fit for the total years the respondent coded
sns.regplot(avgYearsCode['YearsCode'],avgYearsCode['ConvertedComp'])
plt.xlabel("Years of Coding")
plt.ylabel("Salary (USD)")
# Scatter plot with line of best fit for the total years the respondent coded professionally
sns.regplot(avgYearsPro['YearsCodePro'],avgYearsPro['ConvertedComp'])
plt.xlabel("Years of Professional Coding")
plt.ylabel("Salary (USD)")<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.10807390534042015, 'fraction_numerical': 0.023032143761073147, 'mean_word_length': 7.232291666666667, 'pattern_counts': {'":': 0, '<': 32, '<?xml version=': 0, '>': 33, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '8', 'dir': 'jupyter-structured-clean-dedup', 'id': '275051', 'license': 'CNRI-Python', 'n_tokens_mistral': 2897, 'n_tokens_neox': 2666, 'n_words': 772, 'path': '.ipynb_checkpoints/StackOverflow_2020_Code_Starts-1-checkpoint.ipynb', 'repo_name': 'matthew-young/stackoverflow-2020', 'size': '7902'}