<jupyter_start><jupyter_text>Read Data<jupyter_code>data_file = Path.cwd().parent / 'data' / 'kaggle' / 'dataExpLang' / 'base-race-dataset-with-exp-lang.csv'

df = pd.read_csv(data_file, index_col='Match_ID')
df.columns = (
    'cv_id',
    'experience', 
    'french_level',
    'status',
    'cv_received',
    'cv_validated',
    'cv_approved',
    'match_status',
    'response_time',
    'decision_time',
    'race'
)
df.drop(columns=['response_time', 'decision_time', 'cv_id'], inplace=True)
df.index.names = ['match_id']

df['cv_received'] = pd.to_datetime(df.cv_received)
df['cv_validated'] = pd.to_datetime(df.cv_validated)
df['cv_approved'] = pd.to_datetime(df.cv_approved)

df.sample(1)<jupyter_output><empty_output><jupyter_text>EDA<jupyter_code>status_go_match = [
    'A01 - Offer Preparation Started',
    'A02 - Offer Sent to Candidate',
    'A03 - Offer accepted',
    'A03 - Offer declined',
    'A03 - Process Cancelled',
    'A04 - Hiring Request Started',
    'A04a – Hiring Started / Collecting signatures',
    'A04b – Hiring Started / Contract signed',
    'A05 - Onboarding',
    'A06 - Offboarding',
    'B01 - Subcontracting Started',
    'B02 - Subcontracting Signed',
    'D01 – Resignation',
    'T01- Terminated',
    'Candidate validated'
]
df_go = df[df['status'].isin(status_go_match)]
no_go_pct = (1 - df_go.shape[0] / df.shape[0]) * 100
print("Input data has only {}/{} ({}%) accepted candidates.".format(
    df_go.shape[0],
    df.shape[0],
    round(df_go.shape[0] / df.shape[0] * 100, 2)
))
print("Since the data is so skewed, we will not use accuracy as a performance measure")<jupyter_output>Input data has only 269/573 (46.95%) accepted candidates.
Since the data is so skewed, we will not use accuracy as a performance measure<jupyter_text>Feature Engineering Add Computed features<jupyter_code>df['response_time'] = (df['cv_validated'] - df['cv_received']).dt.days
df['decision_time'] = (df['cv_approved'] - df['cv_validated']).dt.days
df.sample(1)<jupyter_output><empty_output><jupyter_text>Add Mapped Status Boolean _Go_ -> 1 or _No Go_ -> 0<jupyter_code>status_map = {
    'Rejected': 0,
    'CV refused': 0,
    'A02 - Offer Sent to Candidate': 1,
    'Candidate refused': 0,
    'A03 - Offer declined': 1,
    'D01 – Resignation': 1,
    'A05 - Onboarding': 1,
    'Candidate dropped out': 0,
    'CV dropped out': 0,
    'T01- Terminated': 1,
    'A03 - Process Cancelled': 1,
    'Dropped out': 0,
    'Approved': 0,
    'CV sent to France': 0,
    'Matched': 0,
    'Candidate validated': 1,
    'A01 - Offer Preparation Started': 1,
    'A04b – Hiring Started / Contract signed': 1,
    'A03 - Offer accepted': 1,
    'CV approved': 0,
    'A04 - Hiring Request Started': 1,
    'Sent to Client': 0
}
df['status_mapped'] = df.status.map(status_map).astype(int)
df.sample(1)<jupyter_output><empty_output><jupyter_text>Remove Outliers<jupyter_code>rt_outliers = df.index[df['response_time'] < 0]
dt_outliers = df.index[df['decision_time'] < 0]

df.loc[rt_outliers, 'response_time'] = df.loc[rt_outliers, 'response_time'].fillna(df.response_time.mean())
df.loc[dt_outliers, 'decision_time'] = df.loc[dt_outliers, 'decision_time'].fillna(df.decision_time.mean())<jupyter_output><empty_output><jupyter_text>Augmenting<jupyter_code>df['french_level'] = df['french_level'].fillna('0')
df = df[df['experience'].notnull()]
df.skew() # <- TODO address high skew
ct = pd.crosstab(df['response_time'], df['status_mapped'])
ct.columns = ['Positive Candidate', 'Negative Candidate']
ct.head()<jupyter_output><empty_output><jupyter_text>Reduce skew<jupyter_code># from scipy.stats import norm, skew

# plt.title('Before transformation')
# rt = df.response_time.dropna() + 1
# sns.distplot(df.response_time)

# plt.figure()
# plt.title('After Transformation')
# sns.distplot(rt.apply(np.log), fit=norm)
# from scipy.stats import norm, skew
# from scipy.special import boxcox1p

# plt.title('Before transformation')
# rt = df.decision_time.dropna() + 1
# sns.distplot(df.decision_time)

# plt.figure()
# plt.title('After Transformation')
# sns.distplot(rt.apply(np.log), fit=norm)<jupyter_output><empty_output><jupyter_text>Label encodingHelps us predict on certain categories of data<jupyter_code># Split decision times into groups of 20 day periods
decision_time_splits = np.ceil(df['decision_time'].max() / 20).astype(int)
decision_map = pd.concat(
    pd.Series(str(i + 1), index=range(i * 20, 20 + i * 20)) 
    for i in range(decision_time_splits)
)

df['response_time'] = df['response_time'].map(decision_map)
df['decision_time'] = df['decision_time'].map(decision_map)

# Replace decision and reponse times by their log values
df['decision_time'] = df['decision_time'].astype(float).dropna().apply(np.log)
df['response_time'] = df['response_time'].astype(float).dropna().apply(np.log)

# Rename target/dependent variable
df.rename(index=str, columns={'status_mapped': 'y'}, inplace=True)<jupyter_output><empty_output><jupyter_text>Encode (replace unique values by integers) experience and french level<jupyter_code>le = LabelEncoder()

le.fit(df['experience'].unique())
df['experience'] = le.transform(df['experience'])

le.fit(df['french_level'].unique())
df['french_level'] = le.transform(df['french_level'])
df['french_level'].fillna('0', inplace=True)

le.fit(df['race'].unique())
df['race'] = le.transform(df['race'])<jupyter_output><empty_output><jupyter_text>Remove rows with null values<jupyter_code>response_time_nulls = df.response_time[df.response_time.isnull()].index
decision_time_nulls = df.decision_time[df.decision_time.isnull()].index
french_level_nulls = df.french_level[df.french_level.isnull()].index

indices_to_drop = response_time_nulls.union(decision_time_nulls).union(french_level_nulls)
df.drop(indices_to_drop, inplace=True)<jupyter_output><empty_output><jupyter_text>Prepare for one hot encoding<jupyter_code>experience_unique = df.experience.unique()
french_level_unique = df.french_level.unique()
race_unique = df.race.unique()

experience_unique, french_level_unique, race_unique
experience_map = dict((e, int(bool(e > max(experience_unique) / 2))) for e in experience_unique)
french_level_map = dict((f, int(bool(f + 1 > max(french_level_unique) / 2))) for f in french_level_unique)
sorted(experience_map.items()), sorted(french_level_map.items())
df['experience'] = df['experience'].map(experience_map)
df['french_level'] = df['french_level'].map(experience_map)<jupyter_output><empty_output><jupyter_text>One hot encoding<jupyter_code>for column in ('french_level', 'experience', 'race'):
    dummies = pd.get_dummies(df[column], prefix=f'_{column}')
    df = df.join(dummies, how='outer').drop([column], axis=1)
df.columns.to_list()
df = df[[
    'y', 
    # 'response_time', 
    # 'decision_time', 
    '_french_level_0.0', 
    '_french_level_1.0',
    '_experience_0',
    '_experience_1',
    '_race_0',
    '_race_1'
]]
y = df['y']
X = df.drop(['y', '_experience_0', '_french_level_0.0', '_race_0'], axis=1)<jupyter_output><empty_output><jupyter_text>Hold out & Cross validation<jupyter_code># n_hold_out = 100
# X, y = X[:-n_hold_out], y[:-n_hold_out]
# X_cv, y_cv = X[:n_hold_out], y[:n_hold_out]
# sns.heatmap(df.corr(), cmap='coolwarm', annot=True, fmt='.1f')<jupyter_output><empty_output><jupyter_text>Split, train and predict on test set<jupyter_code>X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.3, random_state=0)

model = XGBClassifier()
model.fit(X_train, y_train)
y_pred = model.predict(X_test)
from pprint import pprint
pprint(classification_report(y_test, y_pred, output_dict=True))
# plot_importance(model)
explainer = shap.Explainer(model)
shap_values = explainer(X)
shap.plots.bar(shap_values)
df_shap = pd.DataFrame(shap_values, columns=X.columns)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.15850845720143517, 'fraction_numerical': 0.01806765761148129, 'mean_word_length': 5.8225524475524475, 'pattern_counts': {'":': 0, '<': 59, '<?xml version=': 0, '>': 60, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '14', 'dir': 'jupyter-structured-clean-dedup', 'id': '204582', 'license': 'Apache-2.0', 'n_tokens_mistral': 3090, 'n_tokens_neox': 2892, 'n_words': 529, 'path': 'xgboost/xgboost-kaggle-gender.ipynb', 'repo_name': 'ponugoti/eaai-xai', 'size': '7804'}