<jupyter_start><jupyter_text>Dataset<jupyter_code>df = pd.read_excel(DATA_PATH)
idx = np.where(df["native-country"] == "Holand-Netherlands")[0]
data = df.to_numpy()

x = data[:, :-1]
x = np.delete(x, idx, axis=0)
y = data[:, -1]
y = np.delete(y, idx, axis=0)

categorical_features = [1, 2, 3, 4, 5, 6, 7, 9]
numerical_features = [0, 8]

print(f"x shape: {x.shape}")
print(f"y shape: {y.shape}")<jupyter_output>x shape: (48841, 10)
y shape: (48841,)<jupyter_text>y-Data<jupyter_code>def one_hot(y):
    return np.array([0 if val == "<=50K" else 1 for val in y], dtype=np.int32)
y = one_hot(y)<jupyter_output><empty_output><jupyter_text>Helper<jupyter_code>def print_grid_cv_results(grid_result):
    print(
        f"Best model score: {grid_result.best_score_} "
        f"Best model params: {grid_result.best_params_} "
    )
    means = grid_result.cv_results_["mean_test_score"]
    stds = grid_result.cv_results_["std_test_score"]
    params = grid_result.cv_results_["params"]

    for mean, std, param in zip(means, stds, params):
        mean = round(mean, 4)
        std = round(std, 4)
        print(f"{mean} (+/- {2 * std}) with: {param}")<jupyter_output><empty_output><jupyter_text>Sklearn Imports<jupyter_code>from sklearn.compose import ColumnTransformer
from sklearn.preprocessing import OrdinalEncoder
from sklearn.preprocessing import OneHotEncoder
from sklearn.preprocessing import StandardScaler
from sklearn.model_selection import train_test_split
from sklearn.ensemble import RandomForestClassifier
from sklearn.pipeline import Pipeline
from sklearn.model_selection import GridSearchCV
x_train, x_test, y_train, y_test = train_test_split(x, y, test_size=0.3)<jupyter_output><empty_output><jupyter_text>Classifier and Params<jupyter_code>params = {
    "classifier__n_estimators": [50, 100, 200],
    "classifier__max_depth": [None, 100, 200]
}

clf = RandomForestClassifier()<jupyter_output><empty_output><jupyter_text>Ordinal Features<jupyter_code>numeric_transformer = Pipeline(
    steps=[
        ('scaler', StandardScaler())
    ]
)

categorical_transformer = Pipeline(
    steps=[
        ('ordinal', OrdinalEncoder())
    ]
)

preprocessor_odinal = ColumnTransformer(
    transformers=[
        ('numeric', numeric_transformer, numerical_features),
        ('categorical', categorical_transformer, categorical_features)
    ]
)
preprocessor_odinal
preprocessor_odinal.fit(x_train)

x_train_ordinal = preprocessor_odinal.transform(x_train)
x_test_ordinal = preprocessor_odinal.transform(x_test)

print(f"Shape of odinal data: {x_train_ordinal.shape}")
print(f"Shape of odinal data: {x_test_ordinal.shape}")
pipe_ordinal = Pipeline(
    steps=[
        ('preprocessor_odinal', preprocessor_odinal),
        ('classifier', clf)
    ]
)
pipe_ordinal
grid_ordinal = GridSearchCV(pipe_ordinal, params, cv=3)
grid_results_ordinal = grid_ordinal.fit(x_train, y_train)
print_grid_cv_results(grid_results_ordinal)<jupyter_output>Best model score: 0.817947817947818 Best model params: {'classifier__max_depth': 200, 'classifier__n_estimators': 200} 
0.8169 (+/- 0.0022) with: {'classifier__max_depth': None, 'classifier__n_estimators': 50}
0.8173 (+/- 0.002) with: {'classifier__max_depth': None, 'classifier__n_estimators': 100}
0.8178 (+/- 0.0016) with: {'classifier__max_depth': None, 'classifier__n_estimators': 200}
0.8174 (+/- 0.0024) with: {'classifier__max_depth': 100, 'classifier__n_estimators': 50}
0.8179 (+/- 0.002) with: {'classifier__max_depth': 100, 'classifier__n_estimators': 100}
0.8179 (+/- 0.0036) with: {'classifier__max_depth': 100, 'classifier__n_estimators': 200}
0.8169 (+/- 0.001) with: {'classifier__max_depth': 200, 'classifier__n_estimators': 50}
0.8174 (+/- 0.0034) with: {'classifier__max_depth': 200, 'classifier__n_estimators': 100}
0.8179 (+/- 0.0012) with: {'classifier__max_depth': 200, 'classifier__n_estimators': 200}<jupyter_text>OneHot Features<jupyter_code>numeric_transformer = Pipeline(
    steps=[
        ('scaler', StandardScaler())
    ]
)

categorical_transformer = Pipeline(
    steps=[
        ('onehot', OneHotEncoder(handle_unknown="ignore", sparse=False))
    ]
)

preprocessor_onehot = ColumnTransformer(
    transformers=[
        ('numeric', numeric_transformer, numerical_features),
        ('categorical', categorical_transformer, categorical_features)
    ]
)
preprocessor_onehot
preprocessor_onehot.fit(x_train)

x_train_onehot = preprocessor_onehot.transform(x_train)
x_test_onehot = preprocessor_onehot.transform(x_test)

print(f"Shape of onehot data: {x_train_onehot.shape}")
print(f"Shape of onehot data: {x_test_onehot.shape}")
pipe_onehot = Pipeline(
    steps=[
        ('preprocessor_onehot', preprocessor_odinal),
        ('classifier', clf)
    ]
)
pipe_onehot
grid_onehot = GridSearchCV(pipe_onehot, params, cv=3)
grid_results_onehot = grid_onehot.fit(x_train, y_train)
print_grid_cv_results(grid_results_onehot)<jupyter_output>Best model score: 0.818094068094068 Best model params: {'classifier__max_depth': 100, 'classifier__n_estimators': 200} 
0.8173 (+/- 0.0016) with: {'classifier__max_depth': None, 'classifier__n_estimators': 50}
0.8167 (+/- 0.0016) with: {'classifier__max_depth': None, 'classifier__n_estimators': 100}
0.8173 (+/- 0.0038) with: {'classifier__max_depth': None, 'classifier__n_estimators': 200}
0.818 (+/- 0.0024) with: {'classifier__max_depth': 100, 'classifier__n_estimators': 50}
0.8177 (+/- 0.005) with: {'classifier__max_depth': 100, 'classifier__n_estimators': 100}
0.8181 (+/- 0.0016) with: {'classifier__max_depth': 100, 'classifier__n_estimators': 200}
0.8163 (+/- 0.002) with: {'classifier__max_depth': 200, 'classifier__n_estimators': 50}
0.8179 (+/- 0.003) with: {'classifier__max_depth': 200, 'classifier__n_estimators': 100}
0.8175 (+/- 0.002) with: {'classifier__max_depth': 200, 'classifier__n_estimators': 200}<jupyter_text>TensorFlow Model<jupyter_code>from tensorflow.keras.layers import Activation
from tensorflow.keras.layers import Dense
from tensorflow.keras.models import Sequential
from tensorflow.keras.optimizers import SGD
y_train = y_train.reshape(-1, 1)
y_test = y_test.reshape(-1, 1)
def build_model(input_dim, output_dim):
    model = Sequential()
    model.add(Dense(units=128, input_dim=input_dim))
    model.add(Activation("relu"))
    model.add(Dense(units=64))
    model.add(Activation("relu"))
    model.add(Dense(units=output_dim))
    model.add(Activation("sigmoid"))
    return model<jupyter_output><empty_output><jupyter_text>Neural Network with Ordinal Features<jupyter_code>model = build_model(
    input_dim=x_test_ordinal.shape[1],
    output_dim=y_train.shape[1]
)

model.compile(
    loss="binary_crossentropy",
    optimizer=SGD(learning_rate=0.001),
    metrics=["binary_accuracy"]
)

history_ordinal = model.fit(
    x=x_train_ordinal,
    y=y_train,
    epochs=20,
    validation_data=(x_test_ordinal, y_test)
)
val_binary_accuracy = history_ordinal.history["val_binary_accuracy"]

plt.plot(range(len(val_binary_accuracy)), val_binary_accuracy)
plt.show()<jupyter_output><empty_output><jupyter_text>Neural Network with OneHot Features<jupyter_code>model = build_model(
    input_dim=x_train_onehot.shape[1],
    output_dim=y_train.shape[1]
)

model.compile(
    loss="binary_crossentropy",
    optimizer=SGD(learning_rate=0.001),
    metrics=["binary_accuracy"]
)

history_onehot = model.fit(
    x=x_train_onehot,
    y=y_train,
    epochs=20,
    validation_data=(x_test_onehot, y_test)
)
val_binary_accuracy = history_onehot.history["val_binary_accuracy"]

plt.plot(range(len(val_binary_accuracy)), val_binary_accuracy)
plt.show()<jupyter_output><empty_output><jupyter_text>Pass in user-data<jupyter_code>pipe_ordinal.fit(x_train, y_train)
score = pipe_ordinal.score(x_test, y_test)

print(f"Score: {score}")
x_sample = [
    25,
    "Private",
    "11th",
    "Never-married",
    "Machine-op-inspct",
    "Own-child",
    "Black",
    "Male",
    40,
    "United-States"
]
y_sample = 0

y_pred_sample = pipe_ordinal.predict([x_sample])

print(f"Pred: {y_pred_sample}")<jupyter_output>Pred: [0]
{'fraction_non_alphanumeric': 0.135698337881419, 'fraction_numerical': 0.04800297692880179, 'mean_word_length': 5.926975945017182, 'pattern_counts': {'":': 2, '<': 42, '<?xml version=': 0, '>': 41, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '11', 'dir': 'jupyter-structured-clean-dedup', 'id': '7194', 'license': 'MIT', 'n_tokens_mistral': 3295, 'n_tokens_neox': 2999, 'n_words': 503, 'path': 'Chapter13_CaseStudies/CaseStudyIncome/USIncome_model4.ipynb', 'repo_name': 'franneck94/UdemyML', 'size': '8062'}