<jupyter_start><jupyter_text>Utility Functions<jupyter_code>def load_classification_data():
    dataset = fetch_covtype(data_home="data")
    data = np.hstack([dataset.data, dataset.target.reshape(-1, 1)])
    col_names = [f"feature_{i}" for i in range(data.shape[-1])]
    col_names[-1] = "target"
    data = pd.DataFrame(data, columns=col_names)
    data["feature_0_cat"] = pd.qcut(data["feature_0"], q=4)
    data["feature_0_cat"] = "feature_0_" + data.feature_0_cat.cat.codes.astype(str)
    test_idx = data.sample(int(0.2 * len(data)), random_state=42).index
    test = data[data.index.isin(test_idx)]
    train = data[~data.index.isin(test_idx)]
    return (train, test, ["target"])

def print_metrics(y_true, y_pred, tag):
    if isinstance(y_true, pd.DataFrame) or isinstance(y_true, pd.Series):
        y_true = y_true.values
    if isinstance(y_pred, pd.DataFrame) or isinstance(y_pred, pd.Series):
        y_pred = y_pred.values
    if y_true.ndim>1:
        y_true=y_true.ravel()
    if y_pred.ndim>1:
        y_pred=y_pred.ravel()
    val_acc = accuracy_score(y_true, y_pred)
    val_f1 = f1_score(y_true, y_pred, average="macro")
    print(f"{tag} Acc: {val_acc} | {tag} F1: {val_f1}")<jupyter_output><empty_output><jupyter_text>Load Forest Cover Data<jupyter_code>train, test, target_col = load_classification_data()
train, val = train_test_split(train, random_state=42)
cat_col_names = ["feature_0_cat"]
num_col_names = [col for col in train.columns if col not in cat_col_names+target_col]
encoder = ce.OneHotEncoder(cols=cat_col_names)
train_transform = encoder.fit_transform(train)
val_transform = encoder.transform(val)
test_transform = encoder.transform(test)<jupyter_output>D:\miniconda3\envs\df_encoder\lib\site-packages\category_encoders\utils.py:21: FutureWarning: is_categorical is deprecated and will be removed in a future version.  Use is_categorical_dtype instead
  elif pd.api.types.is_categorical(cols):<jupyter_text>BaselineLet's use the default LightGBM model as a baseline.<jupyter_code>clf = lgb.LGBMClassifier(random_state=42, n_jobs=-1)
clf.fit(train_transform.drop(columns=target_col), train_transform[target_col].values.ravel())
val_pred = clf.predict(val_transform.drop(columns=target_col))
print_metrics(val_transform[target_col], val_pred, "Validation")
test_pred = clf.predict(test_transform.drop(columns='target'))
print_metrics(test_transform[target_col], test_pred, "Holdout")<jupyter_output>Validation Acc: 0.8528953641472251 | Validation F1: 0.825508819288814
Holdout Acc: 0.8517409338909829 | Holdout F1: 0.8175438711213123<jupyter_text>CategoryEmbedding Model<jupyter_code>from pytorch_tabular import TabularModel
from pytorch_tabular.models import CategoryEmbeddingModelConfig, NodeConfig, TabNetModelConfig
from pytorch_tabular.config import DataConfig, OptimizerConfig, TrainerConfig, ExperimentConfig
from pytorch_tabular.categorical_encoders import CategoricalEmbeddingTransformer
data_config = DataConfig(
    target=target_col, #target should always be a list. Multi-targets are only supported for regression. Multi-Task Classification is not implemented
    continuous_cols=num_col_names,
    categorical_cols=cat_col_names,
    continuous_feature_transform="quantile_normal",
    normalize_continuous_features=True
)
trainer_config = TrainerConfig(
    auto_lr_find=True, # Runs the LRFinder to automatically derive a learning rate
    batch_size=1024,
    max_epochs=1000,
    gpus=-1,  #index of the GPU to use. -1 means all available GPUs, None, means CPU
)
optimizer_config = OptimizerConfig()
model_config = CategoryEmbeddingModelConfig(
    task="classification",
    layers="4096-4096-512",  # Number of nodes in each layer
    activation="LeakyReLU", # Activation between each layers
    learning_rate = 1e-3,
    metrics=["accuracy", "f1"],
    metrics_params=[{},{"average":"micro"}]
)
tabular_model = TabularModel(
    data_config=data_config,
    model_config=model_config,
    optimizer_config=optimizer_config,
    trainer_config=trainer_config,
)
tabular_model.fit(train=train, test=test)
result = tabular_model.evaluate(test)
print(result)<jupyter_output><empty_output><jupyter_text>To get the prediction as a dataframe, we can use the `predict` method. This will add predictions to the same dataframe that was passed in. For classification problems, we get both the probabilities and the final prediction taking 0.5 as the threshold<jupyter_code>pred_df = tabular_model.predict(test)
pred_df.head()
print_metrics(test['target'], pred_df["prediction"], tag="Holdout")<jupyter_output>Holdout Acc: 0.6147828780915991 | Holdout F1: 0.3269562480388109<jupyter_text>Extract the Learned EmbeddingFor the models that support (CategoryEmbeddingModel and CategoryEmbeddingNODE), we can extract the learned embeddings into a sci-kit learn style Transformer. You can use this in your Sci-kit Learn pipelines and workflows as a drop in replacement.<jupyter_code>transformer = CategoricalEmbeddingTransformer(tabular_model)
train_transform = transformer.fit_transform(train)
clf = lgb.LGBMClassifier(random_state=42)
clf.fit(train_transform.drop(columns='target'), train_transform['target'])
val_transform = transformer.transform(val)
val_pred = clf.predict(val_transform.drop(columns=target_col))
print_metrics(val_transform[target_col], val_pred, "Validation")
test_transform = transformer.transform(test)
test_pred = clf.predict(test_transform.drop(columns=target_col))
print_metrics(test_transform[target_col], test_pred, "Holdout")<jupyter_output>Validation Acc: 0.8561396865829626 | Validation F1: 0.8260076319996745
Holdout Acc: 0.8555876835166348 | Holdout F1: 0.8233005227790506
{'fraction_non_alphanumeric': 0.10267303947601346, 'fraction_numerical': 0.040892193308550186, 'mean_word_length': 6.729138166894665, 'pattern_counts': {'":': 1, '<': 21, '<?xml version=': 0, '>': 23, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '6', 'dir': 'jupyter-structured-clean-dedup', 'id': '360695', 'license': 'MIT', 'n_tokens_mistral': 2037, 'n_tokens_neox': 1876, 'n_words': 386, 'path': 'docs/tutorials/03-Extracting and Using Learned Embeddings.ipynb', 'repo_name': 'jxtrbtk/pytorch_tabular', 'size': '5649'}