<jupyter_start><jupyter_text>Utility Functions<jupyter_code>def make_mixed_classification(n_samples, n_features, n_categories):
    X,y = make_classification(n_samples=n_samples, n_features=n_features, random_state=42, n_informative=5)
    cat_cols = random.choices(list(range(X.shape[-1])),k=n_categories)
    num_cols = [i for i in range(X.shape[-1]) if i not in cat_cols]
    for col in cat_cols:
        X[:,col] = pd.qcut(X[:,col], q=4).codes.astype(int)
    col_names = [] 
    num_col_names=[]
    cat_col_names=[]
    for i in range(X.shape[-1]):
        if i in cat_cols:
            col_names.append(f"cat_col_{i}")
            cat_col_names.append(f"cat_col_{i}")
        if i in num_cols:
            col_names.append(f"num_col_{i}")
            num_col_names.append(f"num_col_{i}")
    X = pd.DataFrame(X, columns=col_names)
    y = pd.Series(y, name="target")
    data = X.join(y)
    return data, cat_col_names, num_col_names

def print_metrics(y_true, y_pred, tag):
    if isinstance(y_true, pd.DataFrame) or isinstance(y_true, pd.Series):
        y_true = y_true.values
    if isinstance(y_pred, pd.DataFrame) or isinstance(y_pred, pd.Series):
        y_pred = y_pred.values
    if y_true.ndim>1:
        y_true=y_true.ravel()
    if y_pred.ndim>1:
        y_pred=y_pred.ravel()
    val_acc = accuracy_score(y_true, y_pred)
    val_f1 = f1_score(y_true, y_pred)
    print(f"{tag} Acc: {val_acc} | {tag} F1: {val_f1}")<jupyter_output><empty_output><jupyter_text>Generate Synthetic Data First of all, let's create a synthetic data which is a mix of numerical and categorical features<jupyter_code>data, cat_col_names, num_col_names = make_mixed_classification(n_samples=10000, n_features=20, n_categories=4)
train, test = train_test_split(data, random_state=42)
train, val = train_test_split(train, random_state=42)<jupyter_output><empty_output><jupyter_text>Importing the Library<jupyter_code>from pytorch_tabular import TabularModel
from pytorch_tabular.models import CategoryEmbeddingModelConfig, NodeConfig, TabNetModelConfig
from pytorch_tabular.config import DataConfig, OptimizerConfig, TrainerConfig, ExperimentConfig<jupyter_output><empty_output><jupyter_text>Category Embedding Model<jupyter_code>data_config = DataConfig(
    target=['target'], #target should always be a list. Multi-targets are only supported for regression. Multi-Task Classification is not implemented
    continuous_cols=num_col_names,
    categorical_cols=cat_col_names,
)
trainer_config = TrainerConfig(
    auto_lr_find=True, # Runs the LRFinder to automatically derive a learning rate
    batch_size=1024,
    max_epochs=100,
    gpus=-1,  #index of the GPU to use. -1 means all available GPUs, None, means CPU
)
optimizer_config = OptimizerConfig()

model_config = CategoryEmbeddingModelConfig(
    task="classification",
    layers="1024-512-512",  # Number of nodes in each layer
    activation="LeakyReLU", # Activation between each layers
    learning_rate = 1e-3
)

experiment_config = ExperimentConfig(project_name="PyTorch Tabular Example", run_name="synthetic_classification_cat_embedding", exp_watch="gradients", log_target="wandb", log_logits=True)
tabular_model = TabularModel(
    data_config=data_config,
    model_config=model_config,
    optimizer_config=optimizer_config,
    trainer_config=trainer_config,
    experiment_config=experiment_config
)
tabular_model.fit(train=train, validation=val)
result = tabular_model.evaluate(test)<jupyter_output><empty_output><jupyter_text>Node Model<jupyter_code>data_config = DataConfig(
    target=['target'], #target should always be a list. Multi-targets are only supported for regression. Multi-Task Classification is not implemented
    continuous_cols=num_col_names,
    categorical_cols=cat_col_names,
)
trainer_config = TrainerConfig(
    auto_lr_find=False, # Runs the LRFinder to automatically derive a learning rate
    batch_size=64,
    accumulate_grad_batches=16,
    max_epochs=100,
    gpus=-1,  #index of the GPU to use. -1 means all available GPUs, None, means CPU
)
optimizer_config = OptimizerConfig()

model_config = NodeConfig(
    task="classification",
    learning_rate = 1e-3
)

experiment_config = ExperimentConfig(project_name="PyTorch Tabular Example", 
                                     run_name="synthetic_classification_node", 
                                     exp_watch="gradients", 
                                     log_target="wandb", 
                                     log_logits=True)
tabular_model = TabularModel(
    data_config=data_config,
    model_config=model_config,
    optimizer_config=optimizer_config,
    trainer_config=trainer_config,
    experiment_config=experiment_config
)
tabular_model.fit(train=train, validation=val)
result = tabular_model.evaluate(test)<jupyter_output><empty_output><jupyter_text>Node Model with Category Embeddings<jupyter_code>data_config = DataConfig(
    target=['target'], #target should always be a list. Multi-targets are only supported for regression. Multi-Task Classification is not implemented
    continuous_cols=num_col_names,
    categorical_cols=cat_col_names,
)
trainer_config = TrainerConfig(
    auto_lr_find=False, # Runs the LRFinder to automatically derive a learning rate
    batch_size=64,
#     accumulate_grad_batches=16,
    max_epochs=100,
    min_epochs=10,
    gpus=-1,  #index of the GPU to use. -1 means all available GPUs, None, means CPU
)
optimizer_config = OptimizerConfig()

model_config = NodeConfig(
    task="classification",
    learning_rate = 1e-3,
    embed_categorical=True
)

experiment_config = ExperimentConfig(project_name="PyTorch Tabular Example", 
                                     run_name="synthetic_classification_node_cat_embed", 
                                     exp_watch="gradients", 
                                     log_target="wandb", 
                                     log_logits=True)
tabular_model = TabularModel(
    data_config=data_config,
    model_config=model_config,
    optimizer_config=optimizer_config,
    trainer_config=trainer_config,
    experiment_config=experiment_config
)
tabular_model.fit(train=train, validation=val)
result = tabular_model.evaluate(test)<jupyter_output><empty_output><jupyter_text>TabNet Model<jupyter_code>data_config = DataConfig(
    target=['target'], #target should always be a list. Multi-targets are only supported for regression. Multi-Task Classification is not implemented
    continuous_cols=num_col_names,
    categorical_cols=cat_col_names,
)
trainer_config = TrainerConfig(
    auto_lr_find=True, # Runs the LRFinder to automatically derive a learning rate
    batch_size=2048,
#     accumulate_grad_batches=16,
    max_epochs=50,
    min_epochs=10,
    early_stopping=None,
    gpus=-1,  #index of the GPU to use. -1 means all available GPUs, None, means CPU
)
optimizer_config = OptimizerConfig()

model_config = TabNetModelConfig(
    task="classification",
    learning_rate = 1e-5,
n_d = 16, n_a=16, n_steps=4,
)

experiment_config = ExperimentConfig(project_name="PyTorch Tabular Example", 
                                     run_name="synthetic_classification_tabnet", 
                                     exp_watch="gradients", 
                                     log_target="wandb", 
                                     log_logits=True)
tabular_model = TabularModel(
    data_config=data_config,
    model_config=model_config,
    optimizer_config=optimizer_config,
    trainer_config=trainer_config,
    experiment_config=experiment_config
)
tabular_model.fit(train=train, validation=val)
result = tabular_model.evaluate(test)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.08786610878661087, 'fraction_numerical': 0.011637029288702928, 'mean_word_length': 3.9668831168831167, 'pattern_counts': {'":': 0, '<': 29, '<?xml version=': 0, '>': 31, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 12, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '7', 'dir': 'jupyter-structured-clean-dedup', 'id': '481716', 'license': 'MIT', 'n_tokens_mistral': 2510, 'n_tokens_neox': 2429, 'n_words': 484, 'path': 'docs/tutorials/05-Experiment_Tracking.ipynb', 'repo_name': 'jxtrbtk/pytorch_tabular', 'size': '7648'}