<jupyter_start><jupyter_text>Sklearn - Adult Income Classificat> Basics - layout: post - toc: false - comments: false - hide: false - search_exclude: true - categories: [sklearn] - use_math: true Repeating content from http://appliedprogramming.net/machine-learning/adult-income-classification.html<jupyter_code>import os
import urllib
from urllib.request import urlretrieve
from pathlib import Path

from sklearn.tree import DecisionTreeClassifier, export_graphviz
import pandas as pd
import numpy as np
from sklearn.model_selection import train_test_split
from sklearn import metrics
from sklearn.model_selection import cross_val_score, GridSearchCV
from sklearn.ensemble import RandomForestClassifier
from sklearn.naive_bayes import BernoulliNB
from sklearn.neighbors import KNeighborsClassifier
from sklearn.svm import SVC
from time import time
from sklearn.pipeline import Pipeline
from sklearn.metrics import roc_auc_score, classification_report
# download the data
base_url = 'https://archive.ics.uci.edu/ml/machine-learning-databases/adult'
filename = 'adult.data'
urlretrieve(f'{base_url}/{filename}', f'{filename}')
cols = ["Age", "Work-Class", "fnlwgt", "Education", "Education-Num", "Marital-Status", "Occupation", 
        "Relationship", "Race", "Sex", "Capital-gain", "Capital-loss", "Hours-per-week", "Native-Country", "Earnings-Raw"]
df = pd.read_csv(filename, header=None, names=cols)
print(f'df.shape - {df.shape}')
df.head(3)
df.replace([' <=50K', ' >50K'], [0, 1], inplace=True)
df.dropna(how='all', inplace=True)<jupyter_output><empty_output><jupyter_text>Selecting K Best features<jupyter_code>x_cols = ["Age", "Education-Num", "Capital-gain", "Capital-loss", "Hours-per-week"]
x_train_np = df[x_cols].values
y_train_np = df['Earnings-Raw'].values

print(f'x_train_np.shape - {x_train_np.shape}. y_train_np.shape - {y_train_np.shape}')<jupyter_output>x_train_np.shape - (32561, 5). y_train_np.shape - (32561,)<jupyter_text>Now we create our transfomer using the chi2 function and a SelectKBest transformer<jupyter_code>from sklearn.feature_selection import SelectKBest
from sklearn.feature_selection import chi2

transformer = SelectKBest(score_func=chi2, k=3)<jupyter_output><empty_output><jupyter_text>This will create a new dataset with reduced number of features<jupyter_code>x_chi2_train_np = transformer.fit_transform(x_train_np, y_train_np)
print(f'x_chi2_train_np.shape - {x_chi2_train_np.shape}')
print(f'chi square scores - {transformer.scores_}')
print(f'selected columns are {np.argsort(transformer.scores_)[-3:]}')<jupyter_output>x_chi2_train_np.shape - (32561, 3)
chi square scores - [0.2340371  0.33515395 0.22332882 0.15052631 0.22968907]
selected columns are [4 0 1]<jupyter_text>We could also implement other correlations such as Pearsons correlation coefficient. As the default scipy pearsonr function accepts only a one dimensional array, we will create a wrapper around it.<jupyter_code>from scipy.stats import pearsonr

def multivariate_pearsonr(x, y):
    scores, pvalues = [], []
    for column in range(x.shape[1]):
        cur_score, cur_p = pearsonr(x[:, column], y)
        scores.append(abs(cur_score))
        pvalues.append(cur_p)
    return (np.array(scores),np.array(pvalues))<jupyter_output><empty_output><jupyter_text>The pearson value could be between -1 and 1.<jupyter_code>transformer = SelectKBest(score_func=multivariate_pearsonr, k=3)
x_pearson_train_np = transformer.fit_transform(x_train_np, y_train_np)
print(transformer.scores_)
print(f'selected columns are {np.argsort(transformer.scores_)[-3:]}')<jupyter_output>[0.2340371  0.33515395 0.22332882 0.15052631 0.22968907]
selected columns are [4 0 1]<jupyter_text>Let us now fit the classifier<jupyter_code>from sklearn.tree import DecisionTreeClassifier

clf = DecisionTreeClassifier()
scores_chi2 = cross_val_score(clf, x_chi2_train_np, y_train_np, scoring='accuracy')
scores_pearson = cross_val_score(clf, x_pearson_train_np, y_train_np, scoring='accuracy')

print("Chi2 performance: {0:.3f}".format(scores_chi2.mean()))
print("Pearson performance: {0:.3f}".format(scores_pearson.mean()))<jupyter_output>Chi2 performance: 0.773
Pearson performance: 0.773<jupyter_text>Unlike in the blog, we got the same output for selectkbest. Hence the performance is same for both  Principal Component Analysis<jupyter_code>from sklearn.decomposition import PCA

pca = PCA(n_components=5)
xd = pca.fit_transform(x_train_np)
print(f'xd.shape - {xd.shape}')

np.set_printoptions(precision=3, suppress=True)
pca.explained_variance_ratio_<jupyter_output>xd.shape - (32561, 5)<jupyter_text>This shows us that the first feature accounts for 99.7 percent of the variance in the dataset<jupyter_code># fit with full dataset
clf = DecisionTreeClassifier(random_state=14)
original_scores = cross_val_score(clf, x_train_np, y_train_np, scoring='accuracy')

print("The average score from the original dataset is {:.4f}".format(np.mean(original_scores)))

# fit with reduced features 
x_train_np.shape<jupyter_output><empty_output><jupyter_text>Lets convert the categorical features into numeric values<jupyter_code>categorical_features = []
for col in df.columns:
    if df[col].dtype != np.int64:
        categorical_features.append(col)
        
categorical_features<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.11092214663643235, 'fraction_numerical': 0.031179138321995464, 'mean_word_length': 6.8067846607669615, 'pattern_counts': {'":': 0, '<': 37, '<?xml version=': 0, '>': 38, 'https://': 1, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 1, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '10', 'dir': 'jupyter-structured-clean-dedup', 'id': '409897', 'license': 'Apache-2.0', 'n_tokens_mistral': 1971, 'n_tokens_neox': 1811, 'n_words': 409, 'path': '_notebooks/2022-01-10-sklearn-2.ipynb', 'repo_name': 'achinta/notes', 'size': '5292'}