<jupyter_start><jupyter_text>Uses the [Census Income Data Set](http://archive.ics.uci.edu/ml/datasets/Census+Income).<jupyter_code>import numpy as np
import pandas as pd
from sklearn.tree import DecisionTreeClassifier
from sklearn.model_selection import GridSearchCV
columns = [
    'age', 'workclass', 'fnlwgt', 'education',
    'education-num', 'marital-status',
    'occupation', 'relationship', 'race',
    'sex', 'capital-gain', 'capital-loss',
    'hours-per-week', 'native-country', 'label'
]
url = 'http://archive.ics.uci.edu/ml/machine-learning-databases/adult'
df_train = pd.read_csv(f'{url}/adult.data', names=columns, na_values='?', sep=', ')
df_test = pd.read_csv(f'{url}/adult.test', names=columns, na_values='?', sep=', ')

df_train.dropna(how='any', inplace=True)
df_test.dropna(how='any', inplace=True)
df_train.shape
df_test.shape
num_train = df_train.shape[0]
df_train.head()
df_test.head()
df_test['label'].replace({ '<=50K.': 0, '>50K.': 1 }, inplace=True)
df_train['label'].replace({ '<=50K': 0, '>50K': 1 }, inplace=True)
df_train.head()
df_test.head()
df = pd.concat([df_train, df_test])
df_one_hot = pd.get_dummies(df)
df_train_one_hot = df_one_hot[:num_train]
df_test_one_hot = df_one_hot[num_train:]
df_train_one_hot.shape[0] == num_train
X_train = df_train_one_hot.drop(columns=['label']).values
y_train = df_train_one_hot['label'].values
X_test = df_test_one_hot.drop(columns=['label']).values
y_test = df_test_one_hot['label'].values
X_train.shape[0] == num_train
parameters = {
    'criterion': ['gini', 'entropy'],
    'splitter': ['best', 'random'],
    'min_samples_split': [2, 4, 8, 16, 32]
}
clf = GridSearchCV(DecisionTreeClassifier(), parameters, scoring='f1_weighted')
clf.fit(X_train, y_train)
print(clf.best_score_)
print(clf.best_params_)
train_preds = np.where(clf.predict(X_train) > 0.5, 1, 0)
(train_preds == y_train).sum() / num_train
probs = clf.predict(X_test)
predictions = np.where(probs > 0.5, 1, 0)
df_test['prediction'] = predictions
df_test['label'].replace({ 0: '<=50K', 1: '>50K' }, inplace=True)
df_test['prediction'].replace({ 0: '<=50K', 1: '>50K' }, inplace=True)
df_test.to_csv('census-income.csv', index=False, float_format='%.f')<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.19126912691269127, 'fraction_numerical': 0.01935193519351935, 'mean_word_length': 6.692041522491349, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 11, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '533909', 'license': 'MIT', 'n_tokens_mistral': 923, 'n_tokens_neox': 878, 'n_words': 147, 'path': 'demo_data/census.ipynb', 'repo_name': 'nyuvis/SliceLens', 'size': '2222'}