<jupyter_start><jupyter_text>Exploratory Data Analysis and Feature Engineering<NAME>, PhDTo learn more about Python, refeer to the following websites* Python : www.python.org* W3Schools : www.w3schools.com/pythonTo learn more about the Python packages we explore in this notebook, refeer to the following websites* NumPy : www.numpy.org* Matplotlib : www.matplotlib.org* Pandas : https://pandas.pydata.org* Scikit-Learn : https://scikit-learn.org/* Seaborn: https://seaborn.pydata.org/* StatsModel : https://www.statsmodels.org<jupyter_code>import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
import seaborn as sns<jupyter_output><empty_output><jupyter_text>Load Data<jupyter_code>file_name = 'https://raw.githubusercontent.com/SumuduTennakoon/MLFoundations/main/Datasets/income_data.csv'

# Load CSV File
data = pd.read_csv(file_name)
data.sample(20)
data.shape
data.describe(include='all').transpose()
data.describe().transpose()
data.columns
data.dtypes<jupyter_output><empty_output><jupyter_text>Drop Unnecessary Columns<jupyter_code>data.drop(labels='Unnamed: 0', axis=1, inplace=True)
data.drop(labels=1, axis=0)
data.head()<jupyter_output><empty_output><jupyter_text>Quering Data<jupyter_code>data.query("age < 60 and age > 30")<jupyter_output><empty_output><jupyter_text>Create Subset of DataFrame macthing a condition<jupyter_code>data_age_90 = data.query("age==90")
data_age_90.shape<jupyter_output><empty_output><jupyter_text>Create a New Column exlauating Logical Expression<jupyter_code>data['is_age_gt_30'] = data['age']>30
data[['age','is_age_gt_30']].head()
data.loc[data['age']==90].head()<jupyter_output><empty_output><jupyter_text>Drop Rows with Missing Values<jupyter_code>data['class'].isna()
data.loc[data['class'].isna()]
# Numpy NaN
a = np.NaN
data.dropna(how='any', axis=0, inplace=True)
data.loc[data['class'].isna()]
data.shape<jupyter_output><empty_output><jupyter_text>Create Unique ID<jupyter_code>data.head()
data.index
data['ID'] = data.index+1
data.head()<jupyter_output><empty_output><jupyter_text>Rearranging Columns<jupyter_code>data.columns
data = data[['ID', 'age', 'workclass', 'education', 'education_num',
       'marital_status', 'occupation', 'relationship', 'race', 'sex',
       'capital_gain', 'capital_loss', 'hours_per_week', 'native_country',
       'class']]

data.head()
data.describe(include='all').transpose()<jupyter_output><empty_output><jupyter_text>Descriptive Statistics For Numerical Columns<jupyter_code># mean
# median
# min/max
# std
data['age'].describe()
f,ax0 = plt.subplots()
ax1 = ax0.twinx()

data['age'].plot(kind='hist', bins=10, ax=ax0)
data['age'].plot(kind='kde', c='red', ax=ax1)
sns.displot(kind='hist', x='age', data=data, bins=10, kde=True)
data['age'].plot(kind='box')
sns.boxplot(y="age", data=data)<jupyter_output><empty_output><jupyter_text>Exploring Categorical Variables<jupyter_code>data['education'].head()
edu_vals = data['education'].unique()
edu_vals
len(edu_vals)
data['education'].nunique()
data['education'].value_counts()
data['education'].value_counts().plot(kind='bar')
edu_values = data['education'].value_counts()
edu_values
type(edu_values)
edu_values.index
edu_values.values
data.head().values
data['education'].describe()
data['age'].max()
data['scaled_age'] = data['age']/data['age'].max()
data['scaled_age']<jupyter_output><empty_output><jupyter_text>Value Counts<jupyter_code>data['education'].value_counts()<jupyter_output><empty_output><jupyter_text>Group By<jupyter_code>data.groupby(by='education')['ID'].count()
data.groupby(by='education')['age'].min()
data.groupby(by='education')['age'].max()
data.groupby(by='education').agg({'age': ['min', 'max', 'mean', 'std', 'sum']})
data['workclass'].head()
data['workclass'].unique()
data.groupby(by=['education', 'workclass']).agg({'age': ['min', 'max', 'mean']})<jupyter_output><empty_output><jupyter_text>Formulating ML Problem to Solve<jupyter_code>data.head()
data['class'].unique()
# <=50K -> 0
# >50K  -> 1
# y = f(X)<jupyter_output><empty_output><jupyter_text>Replace Values (Method 1)<jupyter_code>data.loc[data['class']==' <=50K.', 'class'] = ' <=50K'
data.loc[data['class']==' >50K.', 'class'] = ' >50K'<jupyter_output><empty_output><jupyter_text>Replace Values (Method 2/recomended)<jupyter_code>data['class'].replace(' >50K.', ' >50K')
data['class'].replace(' <=50K.', ' <=50K')
data['class'].unique()<jupyter_output><empty_output><jupyter_text>Transform into Binary Column<jupyter_code>data['y_act'] = np.where(data['class']==' >50K',1,0)
data[['class', 'y_act']].sample(10)
data[['class', 'y_act']].value_counts()
data.shape[0]
# value counts as a ratio
data[['class', 'y_act']].value_counts(normalize=True)<jupyter_output><empty_output><jupyter_text>Explore X Variables<jupyter_code>data.columns
data.head()
data.groupby(by=['class', 'y_act']).agg({'age': ['min', 'max', 'mean', 'std']})<jupyter_output><empty_output><jupyter_text>Select Columns based on Data Type<jupyter_code>data.select_dtypes(include=['float64'])
data.select_dtypes(exclude=['object'])<jupyter_output><empty_output><jupyter_text>Continious Variable to Categorical Variable<jupyter_code>data['age'].hist()
data['age'].describe()
data['age'].value_counts(bins=10)
data['age'].value_counts(bins=10, normalize=True)<jupyter_output><empty_output><jupyter_text>Use `cut()` function<jupyter_code>labels = ['<20', '20-30', '30-40', '40-50', '50-60', '>60']
bin_edges = [0, 20, 30, 40, 50, 60, np.inf]
len(labels)
len(bin_edges)
data['age_group'] = pd.cut(x=data['age'], bins=bin_edges, labels=labels)
data[['age', 'age_group']].sample(10)<jupyter_output><empty_output><jupyter_text>Exercise 1Convert `hours_per_week` into categorical variable<jupyter_code>data['hours_per_week'].describe()
data['hours_per_week'].value_counts(bins=10)
# Type your code here
data.dtypes<jupyter_output><empty_output><jupyter_text>Cross Tables (two-way tables)<jupyter_code>pd.crosstab(data['education'], data['age_group'], margins=True, margins_name='Total')
pd.crosstab(data['education'], data['age_group'], normalize=True).style.background_gradient(cmap='RdYlGn')#.set_precision(5)<jupyter_output><empty_output><jupyter_text>Pivot Table<jupyter_code>pd.pivot_table(data=data, index='education', columns='age_group', values='ID', aggfunc='count', margins=True, margins_name='Total')<jupyter_output><empty_output><jupyter_text>Combine Values of Categorical Variables (Reduce Number of Categories)<jupyter_code>data.education.unique()
data.education_num.unique()
data['education_grp'] = data['education'].replace(' Preschool', 'school')
data['education_group'] = None
data.loc[data['education']==' Preschool', 'education_group'] = 'school'
data.loc[data['education']==' 5th-6th', 'education_group'] = 'school'
data.loc[data['education']==' 1st-4th', 'education_group'] = 'school'
data.loc[data['education']==' 7th-8th', 'education_group'] = 'school'
data.loc[data['education']==' 9th', 'education_group'] = 'school'

data.loc[data['education']==' Some-college', 'education_group'] = 'h. study'
data.loc[data['education']==' Assoc-voc', 'education_group'] = 'h. study'
data.loc[data['education']==' Assoc-acdm', 'education_group'] = 'h. study'
data.loc[data['education']==' Doctorate', 'education_group'] = 'h. study'
data.loc[data['education']==' Prof-school', 'education_group'] = 'h. study'
data.loc[data['education']==' Bachelors', 'education_group'] = 'h. study'
data.loc[data['education']==' HS-grad', 'education_group'] = 'h. study'
data.loc[data['education']==' Masters', 'education_group'] = 'h. study'

data.loc[data['education']==' 10th', 'education_group'] = 'h. school'
data.loc[data['education']==' 11th', 'education_group'] = 'h. school'
data.loc[data['education']==' 12th', 'education_group'] = 'h. school'
data['education_group'].unique()
data.loc[data['education_group'].isna()]
pd.crosstab(data['education_group'], data['age_group'], margins=True, margins_name='Total')<jupyter_output><empty_output><jupyter_text>Apply Custom Functions over Data Frames<jupyter_code>def merge_education_catergory(education):
  if education in (' Preschool', ' 1st-4th', ' 5th-6th', ' 7th-8th', ' 9th'):
    return 'school'
  elif education in (' 10th', ' 11th', ' 12th', ' HS-grad'):
    return 'h. school'
  elif education in (' Bachelors', ' Masters', ' Some-college', ' Assoc-acdm', ' Assoc-voc', ' Doctorate', ' Prof-school'):
    return 'h. Study'

print(f"merge_education_catergory(' 12th') = '{merge_education_catergory(' 12th')}'")
print(f"merge_education_catergory(' Masters') = '{merge_education_catergory(' Masters')}'")
data['education_group2'] = data['education'].apply(merge_education_catergory)
pd.crosstab(data['education_group2'], data['age_group'], margins=True, margins_name='Total')<jupyter_output><empty_output><jupyter_text>Use String Matching (RegEx)<jupyter_code>data.loc[data['education'].str.match('( Preschool| 1st-4th| 5th-6th| 7th-8th| 9th)'), 'education_group3'] = 'school'
data.loc[data['education'].str.match('( 10th| 11th| 12th| HS-grad)'), 'education_group3'] = 'h. school'
data.loc[data['education'].str.match('( Bachelors| Masters| Some-college| Assoc-acdm| Assoc-voc| Doctorate| Prof-school)'), 'education_group3'] = 'h. Study'
pd.crosstab(data['education_group3'], data['age_group'], margins=True, margins_name='Total')<jupyter_output><empty_output><jupyter_text>Correlation<jupyter_code>correlation_matrix = data[['age',  'hours_per_week', 'education_num','y_act']].corr()
correlation_matrix
sns.heatmap(correlation_matrix)<jupyter_output><empty_output><jupyter_text>Model Building Example<jupyter_code>from sklearn.linear_model import LogisticRegression
from sklearn.model_selection import train_test_split
data.columns<jupyter_output><empty_output><jupyter_text>Identify `X` and `y`<jupyter_code>X_variables = ['age',  'hours_per_week', 'education_num']
data[X_variables].head()
y_varibale = 'y_act'
data[y_varibale].head()
# Two classes
data[y_varibale].unique()
X = data[X_variables].values
X
y = data[y_varibale].values
y<jupyter_output><empty_output><jupyter_text>Train Test Split<jupyter_code>X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.3, random_state=42)

print(F"Train sample size = {len(X_train)}")
print(F"Test sample size  = {len(X_test)}")<jupyter_output>Train sample size = 28765
Test sample size  = 12329<jupyter_text>Create Logistic Regression Model* https://en.wikipedia.org/wiki/Logistic_regression* https://scikit-learn.org/stable/modules/generated/sklearn.linear_model.LogisticRegression.html<jupyter_code>model = LogisticRegression()<jupyter_output><empty_output><jupyter_text>Fit Model<jupyter_code>model.fit(X_train, y_train)<jupyter_output><empty_output><jupyter_text>Predict<jupyter_code>model.predict_proba(X_test)<jupyter_output><empty_output><jupyter_text>Predicted Probability for Class 1<jupyter_code>y_pred_prob = model.predict_proba(X_test)[:, 1]
y_pred_prob
y_pred = model.predict(X_test)
y_pred<jupyter_output><empty_output><jupyter_text>Create Results Data Frame for Test Sample<jupyter_code>test_result = pd.DataFrame(data={'y_act':y_test, 'y_pred':y_pred, 'y_pred_prob':y_pred_prob})
test_result.sample(10)<jupyter_output><empty_output><jupyter_text>Confusion Matrix<jupyter_code>cm = pd.crosstab(test_result['y_act'], test_result['y_pred'], margins=True)
cm
acuracy = (cm.loc[0,0] + cm.loc[1,1] ) / cm.loc['All','All'] 
acuracy
from sklearn.ensemble import RandomForestClassifier
model = RandomForestClassifier(n_estimators=1000)
model.fit(X_train, y_train)
model.predict_proba(X_test)
y_pred = model.predict(X_test)
test_result = pd.DataFrame(data={'y_act':y_test, 'y_pred':y_pred, 'y_pred_prob':y_pred_prob})
test_result.sample(10)
cm = pd.crosstab(test_result['y_act'], test_result['y_pred'], margins=True)
cm
acuracy = (cm.loc[0,0] + cm.loc[1,1] ) / cm.loc['All','All'] 
acuracy<jupyter_output><empty_output><jupyter_text>Last update 2021-10-09 by <NAME><jupyter_code><jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1824579039959789, 'fraction_numerical': 0.016754628466113763, 'mean_word_length': 9.972426470588236, 'pattern_counts': {'":': 0, '<': 161, '<?xml version=': 0, '>': 165, 'https://': 7, 'lorem ipsum': 0, 'www.': 5, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 1, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '38', 'dir': 'jupyter-structured-clean-dedup', 'id': '290983', 'license': 'MIT', 'n_tokens_mistral': 4736, 'n_tokens_neox': 4415, 'n_words': 639, 'path': 'LectureNotebooks/6_ExploratoryDataAnalysis.ipynb', 'repo_name': 'cpchathura/MLFoundations', 'size': '11957'}