<jupyter_start><jupyter_text>3.1 Imports<jupyter_code># import libraries
import numpy as np
import pandas as pd
import seaborn as sns

import matplotlib as mpl
import matplotlib.pyplot as plt
%matplotlib inline

from sklearn.compose import ColumnTransformer
from sklearn.preprocessing import StandardScaler, OneHotEncoder
from sklearn.pipeline import Pipeline
# ignore warnings
import warnings
warnings.filterwarnings('ignore')
# load train data
train = pd.read_csv('data/train_prep_1.csv')
# load test data
test = pd.read_csv('data/test_prep_1.csv')
train.head()
train.tail()
test.head()
test.tail()
train.shape
test.shape
train.info()<jupyter_output><class 'pandas.core.frame.DataFrame'>
RangeIndex: 32462 entries, 0 to 32461
Data columns (total 12 columns):
 #   Column          Non-Null Count  Dtype 
---  ------          --------------  ----- 
 0   age             32462 non-null  int64 
 1   workclass       32462 non-null  object
 2   education       32462 non-null  int64 
 3   marital_status  32462 non-null  object
 4   occupation      32462 non-null  object
 5   relationship    32462 non-null  object
 6   race            32462 non-null  object
 7   sex             32462 non-null  object
 8   capital_change  32462 non-null  object
 9   hours_per_week  32462 non-null  int64 
 10  native_country  32462 non-null  object
 11  target          32462 non-null  int64 
dtypes: int64(4), object(8)
memory usage: 3.0+ MB<jupyter_text>3.2 Observe Distributions<jupyter_code>numerical = ['age','education','hours_per_week']
categorical = ['workclass','marital_status','occupation','relationship',
               'race','sex','capital_change','native_country']
sns.set(rc={'figure.figsize':(20,10)})

for feature in numerical:
    sns.countplot(data=train, x=feature, color='lightcoral')
    plt.show()
# 'age' shows a similiar distribution to the population
# no of data counts decrease with education level
# most data has 40 hours_per_week
sns.set(rc={'figure.figsize':(20,10)})

for feature in categorical:
    sns.countplot(data=train, x=feature, color='lightcoral')
    plt.show()
# 'native_country' has too many categories
# rotate labels for clearer view

sns.set(rc={'figure.figsize':(20,10)})

ax = sns.countplot(data=train, x='native_country', color='lightcoral')
ax.set_xticklabels(ax.get_xticklabels(), rotation=90)
plt.show()
# most data's 'workclass' is 'Private'
# most data's 'race' is 'White'
# most data has no 'capital_change'
# nearly all data's native country is 'United State'<jupyter_output><empty_output><jupyter_text>3.3 Observe Distributions with 'target'<jupyter_code># relationship between 'age' and 'target'

plt.figure(figsize=(20,10))
sns.countplot(data=train, x='age', hue='target', palette='Reds')
# most data with over 50k annual income has an age between 30 to 60
# relationship between 'education' and 'target'

plt.figure(figsize=(20,10))
sns.countplot(data=train, x='education', hue='target', palette='Reds')
# the observation is interesting
# considering the 4 most common education levels: from 0 to 4
# each category has similiar numbers of data with over 50k annual income
# that means with PhD graduates excluded
# high income individuals distribute even across different education levels
# relationship between 'hours_per_week' and 'target'

plt.figure(figsize=(20,10))
sns.countplot(data=train, x='hours_per_week', hue='target', palette='Reds')
# high income is associated with long working hours
# relationship between 'marital_status' and 'target'

plt.figure(figsize=(20,10))
sns.countplot(data=train, x='marital_status', hue='target', palette='Reds')
# married people has higher percentage of over 50k annual income
# we cannot be certain about whether it is correlation or causation relationship
# in this project we only consider the correlation of features and target
# further study is required for causal inference
# relationship between 'race' and 'target'

plt.figure(figsize=(20,10))
sns.countplot(data=train, x='race', hue='target', palette='Reds')
# with sufficient data
# black has larger portion of data with below 50k annual income
# points to note: Asian-Pac-Islander group of data has highest percentage of high income data
# relationship between 'sex' and 'target'

plt.figure(figsize=(20,10))
sns.countplot(data=train, x='sex', hue='target', palette='Reds')
# significantly less female with over 50k annual income
# relationship between 'capital_change' and 'target'

plt.figure(figsize=(20,10))
sns.countplot(data=train, x='capital_change', hue='target', palette='Reds')
# surprisingly, data with capital change has higher probability of high income
# one of the reasons may be wealth people tend to have capital change
# including capital gain and loss<jupyter_output><empty_output><jupyter_text>3.4 Heatmap<jupyter_code># create a heatmap

plt.figure(figsize=(10,10))
sns.heatmap(train.corr(), annot=True)
# there is no strong correlations between numerical features
# 3.5 encoding categorical features

# have to put it in the modelling part
numerical_features = ['age','education','hours_per_week']

numerical_transformer = StandardScaler()
# if more than one step
# use Pipeline(steps=[])
categorical_features = ['workclass','marital_status','occupation','relationship',
               'race','sex','capital_change','native_country']

categorical_transformer = OneHotEncoder(drop='first')
preprocessor = ColumnTransformer(transformers=[
    ('num', numerical_transformer, numerical_features),
    ('cat', categorical_transformer, categorical_features)
])
x = preprocessor.fit_transform(train)
#END<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.11083348206318044, 'fraction_numerical': 0.031768695341781186, 'mean_word_length': 5.205980066445183, 'pattern_counts': {'":': 0, '<': 17, '<?xml version=': 0, '>': 17, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 1, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '4', 'dir': 'jupyter-structured-clean-dedup', 'id': '651794', 'license': 'MIT', 'n_tokens_mistral': 1858, 'n_tokens_neox': 1677, 'n_words': 529, 'path': '3_EDA.ipynb', 'repo_name': 'yatfungleung/Census-Income-Classification', 'size': '5603'}