<jupyter_start><jupyter_text>DOU Salary RegressionPredict the salaries of Senior Automation QA in Kyiv, using the salary dataset from DOU.ua.<jupyter_code>from urllib.request import urlretrieve

import matplotlib.pyplot as plt
import numpy as np
import pandas as pd

from sklearn import linear_model
from sklearn.metrics import r2_score
%matplotlib inline<jupyter_output><empty_output><jupyter_text>Config<jupyter_code>dataset_file_name = "2017_june_final.csv"
download_fresh_data = True<jupyter_output><empty_output><jupyter_text>Prepare Data<jupyter_code># Pulling fresh data from the Internet, if we want to.
if download_fresh_data:
    urlretrieve("https://github.com/imax/dou-salaries/raw/master/data/"
                + dataset_file_name, dataset_file_name)
# Loading the data into memory.
df = pd.read_csv(dataset_file_name)
# Filtering the data.
# Removing the extreme examples, and limiting our scope to one city and position.
df = df[(df["salary"] <= 10000) & 
        (df["exp"] < 10) & 
        (df["Город"] == "Киев") & 
        (df["Должность"] == "Senior QA engineer") &
        (df["Специализация"] == "Automation QA") &
        (df["Тип.компании"] == "Аутсорсинговая")]<jupyter_output><empty_output><jupyter_text>Train Linear Regression<jupyter_code>X = df["exp"].values.reshape(-1, 1)
y = df["salary"].values.reshape(-1, 1)
plt.hist(y)
plt.show()
# any percentile between 0 and 100 can be computed for any collection of values
# and it is always an element of the collection, used np.percentile function
# it automatically sort passed collection to it
print(f'25% percentile: {np.percentile(y, 25)}')
print(f'50% percentile: {np.percentile(y, 50)}')
print(f'75% percentile: {np.percentile(y, 75)}')
dataset_file_name = "2016_may_final.csv"
download_fresh_data = True
# Pulling fresh data from the Internet, if we want to.
if download_fresh_data:
    urlretrieve("https://github.com/imax/dou-salaries/raw/master/data/"
                + dataset_file_name, dataset_file_name)
# Loading the data into memory.
df = pd.read_csv(dataset_file_name)
# Filtering the data.
# Removing the extreme examples, and limiting our scope to one city and position.
df = df[(df["salary"] <= 10000) & 
        (df["exp"] < 10) & 
        (df["Город"] == "Киев") & 
        (df["Должность"] == "Senior QA engineer") &
        (df["Специализация"] == "Automation QA") &
        (df["Тип.компании"] == "Аутсорсинговая")]
X = df["exp"].values.reshape(-1, 1)
y = df["salary"].values.reshape(-1, 1)
plt.hist(y)
plt.show()
print(f'25% percentile: {np.percentile(y, 25)}')
print(f'50% percentile: {np.percentile(y, 50)}')
print(f'75% percentile: {np.percentile(y, 75)}')
# Looks like salary of Senior Automation QA engineer that works in Kyiv's "galera" changed a lot
# we can observe a big salary bump within May 2016 to June 2018 (please observe histograms above), e.g.
# difference between percentiles:
### 25% percentile: 2575 (year 2017) - 2450 (year 2016) = + 125 $
### 50% percentile: 2850 (year 2017) - 2565 (year 2016) = + 285 $
### 75% percentile: 3000 (year 2017) - 2900 (year 2016) = + 100 $
# so average salary observably increased.
# use old May 2016 data to train model
salary_regression = linear_model.LinearRegression()
salary_regression.fit(X, y)<jupyter_output><empty_output><jupyter_text>Plot & Predict New Examples<jupyter_code># Using the trained model to predict salaries for previously unseen data.
# (2.0, 3.5, 4.5 years of experience).
print("New examples:")
X_new_examples = np.array([2.0, 3.5, 4.5]).reshape(-1, 1)
y_new_examples = salary_regression.predict(X_new_examples)
print('\n'.join([
    "{0} yrs. -> ${1:.0f}".format(exp[0], sal[0])
    for exp, sal in zip(X_new_examples, y_new_examples)
]))
# Plotting the data and the regression line.
plt.figure(facecolor="white", figsize=(12, 8))
plt.scatter(
    x=df["exp"],
    y=df["salary"],
    color="black",
    facecolor="white",
    marker="o",
    s=40,
    zorder=2
)
plt.plot(
    X,
    salary_regression.predict(X),
    color="blue",
    alpha=0.5,
    linewidth=2,
    zorder=1
)

# Add newly predicted values to the plot.
plt.scatter(
    x=X_new_examples,
    y=y_new_examples,
    facecolor="red",
    marker="x",
    s=60,
    linewidth=2,
    zorder=2
)

plt.xlim(xmin=-1)
plt.ylim(ymin=0)
plt.xlabel("Years of Experience")
plt.ylabel("Monthly Salary, $")
plt.title("Salaries of Senior Automation QA Engineers in Kyiv due to DOU stats on May 2016")
plt.show()
# use new June 2017 data to train model
dataset_file_name = "2017_june_final.csv"
urlretrieve("https://github.com/imax/dou-salaries/raw/master/data/"
            + dataset_file_name, dataset_file_name)
# Loading the data into memory.
df = pd.read_csv(dataset_file_name)
# Filtering the data.
# Removing the extreme examples, and limiting our scope to one city and position.
df = df[(df["salary"] <= 10000) & 
        (df["exp"] < 10) & 
        (df["Город"] == "Киев") & 
        (df["Должность"] == "Senior QA engineer") &
        (df["Специализация"] == "Automation QA") &
        (df["Тип.компании"] == "Аутсорсинговая")]
X = df["exp"].values.reshape(-1, 1)
y = df["salary"].values.reshape(-1, 1)

salary_regression = linear_model.LinearRegression()
salary_regression.fit(X, y)
# Using the trained model to predict salaries for previously unseen data.
# (2.0, 3.5, 4.5 years of experience).
print("New examples:")
X_new_examples = np.array([2.0, 3.5, 4.5]).reshape(-1, 1)
y_new_examples = salary_regression.predict(X_new_examples)
print('\n'.join([
    "{0} yrs. -> ${1:.0f}".format(exp[0], sal[0])
    for exp, sal in zip(X_new_examples, y_new_examples)
]))
# Plotting the data and the regression line.
plt.figure(facecolor="white", figsize=(12, 8))
plt.scatter(
    x=df["exp"],
    y=df["salary"],
    color="black",
    facecolor="white",
    marker="o",
    s=40,
    zorder=2
)
plt.plot(
    X,
    salary_regression.predict(X),
    color="blue",
    alpha=0.5,
    linewidth=2,
    zorder=1
)

# Add newly predicted values to the plot.
plt.scatter(
    x=X_new_examples,
    y=y_new_examples,
    facecolor="red",
    marker="x",
    s=60,
    linewidth=2,
    zorder=2
)

plt.xlim(xmin=-1)
plt.ylim(ymin=0)
plt.xlabel("Years of Experience")
plt.ylabel("Monthly Salary, $")
plt.title("Salaries of Senior Automation QA Engineers in Kyiv due to DOU stats on June 2017")
plt.show()<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1474147414741474, 'fraction_numerical': 0.036303630363036306, 'mean_word_length': 4.237860082304527, 'pattern_counts': {'":': 0, '<': 27, '<?xml version=': 0, '>': 23, 'https://': 3, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '5', 'dir': 'jupyter-structured-clean-dedup', 'id': '447742', 'license': 'MIT', 'n_tokens_mistral': 2533, 'n_tokens_neox': 2300, 'n_words': 578, 'path': 'dou_salary_regression/dou_salary_regression.ipynb', 'repo_name': 'duboviy/data_science', 'size': '6363'}