<jupyter_start><jupyter_text>Recruiting data analysisIn this notebook we do some basic analysis of the synthetic recruiting data, to understand existing biases before training a model.<jupyter_code>from pathlib import Path

import pandas as pd
from helpers.recruiting import bin_years_experience
from helpers.plot import bar_chart
from helpers import export_plot<jupyter_output><empty_output><jupyter_text>Directory for saving all processed data.<jupyter_code>artifacts_dir = Path("../../artifacts")
# override data_dir in source notebook
# this is stripped out for the hosted notebooks
artifacts_dir = Path("../../../artifacts")<jupyter_output><empty_output><jupyter_text>Load the preprocessed data. Check out the preprocessing notebook for details on how this data was obtained.<jupyter_code>data_dir = artifacts_dir / "data" / "recruiting"

test = pd.read_csv(data_dir / "processed" / "test.csv")
test_raw = pd.read_csv(data_dir / "raw" / "test.csv")<jupyter_output><empty_output><jupyter_text>Bias in dataVisualise biases present in the data<jupyter_code>employed_by_sex = test[["sex_male", "employed_yes"]].groupby("sex_male").mean()

fig_employed_by_sex = bar_chart(
    x=["Female", "Male"],
    y=employed_by_sex.employed_yes,
    title="Propotion of applicants employed by sex",
    ylabel="Proportion",
)
fig_employed_by_sex
export_plot(fig_employed_by_sex, "employed_by_sex.json")
employed_by_race = (
    test[["race_white", "employed_yes"]].groupby("race_white").mean()
)

fig_employed_by_race = bar_chart(
    x=["Black", "White"],
    y=employed_by_race.employed_yes,
    title="Propotion of applicants employed by race",
    ylabel="Proportion",
)
fig_employed_by_race
export_plot(fig_employed_by_race, "employed_by_race.json")<jupyter_output><empty_output><jupyter_text>Possible resolving variablesLet's look at the relationship between years of experience and whether the applicant was employed.<jupyter_code>employed_by_experience = (
    test.assign(exp=test_raw.years_experience.map(bin_years_experience))
    .loc[:, ["exp", "employed_yes"]]
    .groupby("exp")
    .aggregate(["mean", "count"])
    .reset_index()
)

fig_employed_by_experience = bar_chart(
    x=employed_by_experience.exp,
    y=employed_by_experience["employed_yes"]["mean"],
    title="Proportion of applicants employed by years of experience",
    ylabel="Proportion",
    xticks={
        "tickvals": [0, 1, 2, 3],
        "ticktext": ["0-2 years", "3-5 years", "6-9 years", "10+ years"],
    },
)
fig_employed_by_experience
export_plot(
    fig_employed_by_experience, "employed_by_experience.json",
)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.11762466692044157, 'fraction_numerical': 0.004567948229920061, 'mean_word_length': 6.045576407506703, 'pattern_counts': {'":': 2, '<': 21, '<?xml version=': 0, '>': 21, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '5', 'dir': 'jupyter-structured-clean-dedup', 'id': '421553', 'license': 'MIT', 'n_tokens_mistral': 921, 'n_tokens_neox': 848, 'n_words': 189, 'path': 'src/notebooks/recruiting/data.ipynb', 'repo_name': 'CDEIUK/ml-bias', 'size': '2627'}