<jupyter_start><jupyter_text>Clusters + one-hot encoding<jupyter_code># We have the option to encode the variables into the
# clusters they belong to:

X_train_enc["clusters"] = kmeans.fit_predict(X_train_enc[vars_categorical])
X_test_enc["clusters"] = kmeans.predict(X_test_enc[vars_categorical])

X_train_enc["clusters"].head()
# We would then drop the original variables

X_train_enc.drop(labels=vars_categorical, axis=1, inplace=True)
X_test_enc.drop(labels=vars_categorical, axis=1, inplace=True)
# And then perform one-hot encoding of the clusters

ohe = OneHotEncoder(variables = "clusters",  drop_last=False, ignore_format=True)

ohe.fit(X_train_enc)

# let's transform train and test sets
X_train_enc = ohe.transform(X_train_enc)
X_test_enc = ohe.transform(X_test_enc)

# The final dataset
X_train_enc.head()<jupyter_output><empty_output><jupyter_text>Distance to clusters<jupyter_code># let's transform train and test sets

X_train_enc = encoder.transform(X_train)
X_test_enc = encoder.transform(X_test)
# Alternatively we can encode the categorical
# variables into their distances to each of 
# the clusters

clusters = [f"cluster_{i}" for i in range(5)]

X_train_enc[clusters] = kmeans.transform(X_train_enc[vars_categorical])
X_test_enc[clusters] = kmeans.transform(X_test_enc[vars_categorical])

# The final dataset
X_train_enc[clusters].head()
# We would then drop the original variables

X_train_enc.drop(labels=vars_categorical, axis=1, inplace=True)
X_test_enc.drop(labels=vars_categorical, axis=1, inplace=True)

# The final dataset
X_train_enc.head()<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.10056214865708932, 'fraction_numerical': 0.003123048094940662, 'mean_word_length': 8.0, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '2', 'dir': 'jupyter-structured-clean-dedup', 'id': '626944', 'license': 'MIT', 'n_tokens_mistral': 581, 'n_tokens_neox': 570, 'n_words': 121, 'path': 'ch02-categorical-encoding/Recipe-13-Encoding-categories-with-k-means.ipynb', 'repo_name': 'PacktPublishing/Python-Feature-Engineering-Cookbook-Second-Edition', 'size': '1601'}