<jupyter_start><jupyter_text>Load Dataset<jupyter_code>X_train = pd.read_csv('../data/train.csv')
X_train.head()
X_train.columns = X_train.columns.str.replace('-', '_') 

numerical = ['age','fnlwgt','education_num','capital_gain','capital_loss','hours_per_week']
categorical = ['workclass','education','marital_status','occupation','relationship','race','sex','native_country','target']

# Getting rid of blanks at the start of categorical features
X_train[categorical] = X_train[categorical].apply(lambda x: x.str.strip())<jupyter_output><empty_output><jupyter_text>Exploring data<jupyter_code>X_train.describe()
print("The number of unique workclass is: {0}".format(X_train['workclass'].unique().shape[0]))
print("The categories are:\n {0}".format(X_train['workclass'].unique()))
print("The number of unique education is: {0}".format(X_train['education'].unique().shape[0]))
print("The categories are:\n {0}".format(X_train['education'].unique()))
print("The number of unique marital_status is: {0}".format(X_train['marital_status'].unique().shape[0]))
print("The categories are:\n {0}".format(X_train['marital_status'].unique()))
print("The number of unique occupation is: {0}".format(X_train['occupation'].unique().shape[0]))
print("The categories are :\n {0}".format(X_train['occupation'].unique()))
print("The number of unique relationship is: {0}".format(X_train['relationship'].unique().shape[0]))
print("The categories are :\n {0}".format(X_train['relationship'].unique()))
print("The number unique race is: {0}".format(X_train['race'].unique().shape[0]))
print("The categories are:\n {0}".format(X_train['race'].unique()))
print("The number unique sex is: {0}".format(X_train['sex'].unique().shape[0]))
print("The categories are:\n {0}".format(X_train['sex'].unique()))
print("The number of unique native_country is: {0}".format(X_train['native_country'].unique().shape[0]))
print("The countries are:\n {0}".format(X_train['native_country'].unique()))
print("Number of observations with income Class >50K $ = {0}".format(X_train['target'].value_counts()[1]))
print("Number of observations with income Class <=50K $ = {0}".format(X_train['target'].value_counts()[0]))<jupyter_output>Number of observations with income Class >50K $ = 6270
Number of observations with income Class <=50K $ = 19778<jupyter_text>Observations- Categorical variables `native_country`, `occupation` and `workclass` have `?` in some places.- Outliers are present in numerical variables `capital_gain` and `capital_loss`.- Target classes aren't evenly distributed.  Relation among different variables<jupyter_code>X_train.head()
# Plot distribution of age across income level by sex
alt.data_transformers.disable_max_rows()

p1 = alt.Chart(X_train).mark_boxplot().encode(
    alt.X('target:O', title='Income level'),
    alt.Y('age:Q', title='Age (years)'),
    facet='sex:N'
).properties(
    title='Distribution of age across income level by sex',
    width=400,
    height=200
)
p1
# Plot distribution of years in education by income level
p2 = alt.Chart(X_train).mark_bar().encode(
    alt.X('education_num:Q', bin=alt.Bin(step=1), title='Years in education'),
    alt.Y('count()', title='Counts'),
    facet='target:N'
).properties(
    title='Distribution of years in education by income level',
    width=400,
    height=200
)
p2
# Plot distribution of weekly hours of work by income level 
p3 = alt.Chart(X_train).mark_bar().encode(
    alt.X('hours_per_week:Q', bin=alt.Bin(step=2), title='Weekly hours of work'),
    alt.Y('count()', title='Counts'),
    facet='target:N'
).properties(
    title='Distribution of weekly hours of work by income level',
    width=400,
    height=200
)
p3
# Plot distribution of capital gain by income level
p4 = alt.Chart(X_train).mark_bar().encode(
    alt.X('capital_gain:Q', bin=alt.Bin(extent=[0, 500], step=50), title='Capital gain'),
    alt.Y('count()', title='Counts'),
    facet='target:N'
).properties(
    title='Distribution of capital gain by income level',
    width=400,
    height=150
)

# Plot distribution of capital loss by income level
p5 = alt.Chart(X_train).mark_bar().encode(
    alt.X('capital_loss:Q', bin=alt.Bin(extent=[0, 1000], step=50), title='Capital loss'),
    alt.Y('count()', title='Counts'),
    facet='target:N'
).properties(
    title='Distribution of capital loss by income level',
    width=400,
    height=150
)

p4
p5
# Barplot showing frequency of different races by income level 
p6 = alt.Chart(X_train).mark_bar().encode(
    alt.X('race:O', title=None),
    alt.Y('count()', title='Counts'),
    facet='target:N'
).properties(
    title='Frequency of race by income level',
    width=300,
    height=200
)
p6
# Barplot showing frequency of different marital statuses by income level 
p7 = alt.Chart(X_train).mark_bar().encode(
    alt.X('marital_status:O', title=None),
    alt.Y('count()', title='Counts'),
    facet='target:N'
).properties(
    title='Frequency of marital status by income level',
    width=400,
    height=200
)
p7
# Barplot showing frequency of occupation by income level 
p8 = alt.Chart(X_train).mark_bar().encode(
    alt.X('occupation:O', title=None),
    alt.Y('count()', title='Counts'),
    facet='target:N'
).properties(
    title='Frequency of occupation by income level',
    width=400,
    height=200
)
p8<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.17088251826869028, 'fraction_numerical': 0.023234026606707887, 'mean_word_length': 5.852374839537869, 'pattern_counts': {'":': 0, '<': 14, '<?xml version=': 0, '>': 14, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 1, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '3', 'dir': 'jupyter-structured-clean-dedup', 'id': '627094', 'license': 'MIT', 'n_tokens_mistral': 1815, 'n_tokens_neox': 1722, 'n_words': 407, 'path': 'eda/EDA_Adult_data.ipynb', 'repo_name': 'reikookamoto/DSCI_522_group-307', 'size': '5337'}