<jupyter_start><jupyter_text>Poker Hand Induction Datasethttps://www.kaggle.com/c/poker-rule-induction/dataFastai v2 best result: 99,48% (I suspect it could improve, cause the validation error kept slowly decreasing, but I did not have more time to keep training)TabNet best result: 99,10% (Also suspect it could get better)Most pain for fastai was setting hyperparameters (number of layers, number of neurons in each one, and dropouts). For TabNet, the same thing (n_d, n_a, n_steps)Crucial: treat ranks also as categories, not only as continuousBased on the work of:https://github.com/muellerzr/fastai2-Tabular-Baselinesandhttps://github.com/mgrankin/fast_tabnet<jupyter_code>from fastai2.basics import *
from fastai2.tabular.all import *
from fast_tabnet.core import *
df = pd.read_csv('/media/hdd3tb/data/kaggle_pokerhand/train.csv')
df.head()
for i in range(1,6):
    df['cC'+str(i)]=df['C'+str(i)]
cat_vars = ['S1', 'S2', 'S3', 'S4', 'S5','cC1', 'cC2', 'cC3', 'cC4', 'cC5']
cont_vars = ['C1', 'C2', 'C3', 'C4', 'C5']
dep_var = ['hand']
procs = [Categorify, Normalize]
splits = RandomSplitter()(range_of(df))
splits
to = TabularPandas(df, procs, cat_names=cat_vars, cont_names=cont_vars, y_names=dep_var, splits=splits)
to
dls = to.dataloaders(bs=512)
dls.show_batch()
class LabelSmoothingCrossEntropyFlat(BaseLoss):
    "Same as `nn.CrossEntropyLoss`, but flattens input and target."
    y_int = True
    def __init__(self, *args, axis=-1, **kwargs): super().__init__(LabelSmoothingCrossEntropy, *args, axis=axis, **kwargs)
    def decodes(self, x):    return x.argmax(dim=self.axis)
    def activation(self, x): return F.softmax(x, dim=self.axis)
learn = tabular_learner(dls, layers=[100, 50, 50], ps=[0.01, 0.01, 0.02], metrics=accuracy, opt_func=ranger, loss_func=CrossEntropyLossFlat(),n_out=10)
learn.model
learn.lr_find()
learn.fit_flat_cos(10, 5e-1)
learn.fit_flat_cos(10, 1e-1)
learn.fit_flat_cos(10, 1e-1)
learn.fit_flat_cos(10, 1e-1)
learn.fit_flat_cos(10, 1e-1)
learn.fit_flat_cos(10, 1e-1)
learn.fit_flat_cos(10, 1e-1)
learn.fit_flat_cos(10, 1e-1)
learn.fit_flat_cos(10, 1e-1)
learn.fit_flat_cos(10, 1e-1)
learn.fit_flat_cos(10, 1e-1)
learn.fit_flat_cos(10, 1e-1)
learn.fit_flat_cos(10, 1e-1)
learn.fit_flat_cos(10, 1e-1)
learn.fit_flat_cos(10, 1e-1)
learn.save('round2_9922')<jupyter_output><empty_output><jupyter_text>99,22%<jupyter_code>learn.fit_flat_cos(10, 1e-1)
learn.fit_flat_cos(10, 1e-1)
learn.save('round2_9932')<jupyter_output><empty_output><jupyter_text>99,32%<jupyter_code>learn.fit_flat_cos(10, 1e-1)
learn.fit_flat_cos(10, 1e-1)
learn.fit_flat_cos(10, 1e-1)
learn.fit_flat_cos(10, 1e-1)
learn.save('round2_9934')<jupyter_output><empty_output><jupyter_text>99,35%<jupyter_code>learn.fit_flat_cos(10, 1e-1)
learn.fit_flat_cos(10, 1e-1)
learn.fit_flat_cos(10, 1e-1)
learn.fit_flat_cos(10, 1e-1)
learn.fit_flat_cos(10, 1e-1)
learn.fit_flat_cos(10, 1e-1)
learn.fit_flat_cos(10, 1e-1)
learn.fit_flat_cos(10, 1e-1)
learn.fit_flat_cos(10, 1e-1)
learn.fit_flat_cos(10, 1e-1)
learn.save('round2_9940')<jupyter_output><empty_output><jupyter_text>99,40%<jupyter_code>learn.fit_flat_cos(10, 1e-1)
learn.fit_flat_cos(10, 1e-1)
learn.fit_flat_cos(10, 1e-1)
learn.fit_flat_cos(10, 1e-1)
learn.fit_flat_cos(10, 1e-1)
learn.fit_flat_cos(10, 1e-1)
learn.fit_flat_cos(10, 1e-1)
learn.fit_flat_cos(10, 1e-1)
learn.fit_flat_cos(10, 1e-1)
learn.fit_flat_cos(10, 1e-1)
learn.fit_flat_cos(10, 1e-1)
learn.fit_flat_cos(10, 1e-1)
learn.fit_flat_cos(10, 1e-1)
learn.fit_flat_cos(10, 1e-1)
learn.fit_flat_cos(10, 1e-1)
learn.fit_flat_cos(10, 1e-1)
learn.fit_flat_cos(10, 1e-1)
learn.fit_flat_cos(10, 1e-1)
learn.fit_flat_cos(10, 1e-1)
learn.fit_flat_cos(10, 1e-1)
learn.fit_flat_cos(10, 1e-1)
learn.fit_flat_cos(10, 1e-1)
learn.fit_flat_cos(10, 1e-1)
learn.fit_flat_cos(10, 1e-1)
learn.fit_flat_cos(10, 1e-1)
learn.fit_flat_cos(10, 1e-1)
learn.save('round2_9944')<jupyter_output><empty_output><jupyter_text>99,44%<jupyter_code>learn.fit_flat_cos(10, 1e-1)
learn.fit_flat_cos(10, 1e-1)
learn.fit_flat_cos(10, 1e-1)
learn.fit_flat_cos(10, 1e-1)
learn.fit_flat_cos(10, 1e-1)
learn.fit_flat_cos(10, 1e-1)
learn.fit_flat_cos(10, 1e-1)
learn.fit_flat_cos(10, 1e-1)
learn.fit_flat_cos(10, 1e-1)
learn.fit_flat_cos(10, 1e-1)
learn.fit_flat_cos(10, 1e-1)
learn.fit_flat_cos(10, 1e-1)
learn.fit_flat_cos(10, 1e-1)
learn.fit_flat_cos(10, 1e-1)
learn.fit_flat_cos(10, 1e-1)
learn.fit_flat_cos(10, 1e-1)
learn.fit_flat_cos(10, 1e-1)
learn.fit_flat_cos(10, 1e-1)
learn.fit_flat_cos(10, 1e-1)
learn.fit_flat_cos(10, 1e-1)
learn.fit_flat_cos(10, 1e-1)
learn.fit_flat_cos(10, 1e-1)
learn.fit_flat_cos(10, 1e-1)
learn.fit_flat_cos(10, 1e-1)
learn.fit_flat_cos(10, 1e-1)
learn.fit_flat_cos(10, 1e-1)
learn.fit_flat_cos(10, 1e-1)
learn.fit_flat_cos(10, 1e-1)
learn.fit_flat_cos(10, 1e-1)
learn.fit_flat_cos(10, 1e-1)
learn.fit_flat_cos(10, 1e-1)
learn.fit_flat_cos(10, 1e-1)
learn.fit_flat_cos(10, 1e-1)
learn.fit_flat_cos(10, 1e-1)
learn.fit_flat_cos(10, 1e-1)
learn.fit_flat_cos(10, 1e-1)
learn.fit_flat_cos(10, 1e-1)
learn.fit_flat_cos(10, 1e-1)
learn.fit_flat_cos(10, 1e-1)
learn.fit_flat_cos(10, 1e-1)
learn.fit_flat_cos(10, 1e-1)
learn.fit_flat_cos(10, 1e-1)
learn.fit_flat_cos(10, 1e-1)
learn.fit_flat_cos(10, 1e-1)
learn.fit_flat_cos(10, 1e-1)
learn.fit_flat_cos(10, 1e-1)
learn.fit_flat_cos(10, 1e-1)
learn.fit_flat_cos(10, 1e-1)
learn.fit_flat_cos(10, 1e-1)
learn.fit_flat_cos(10, 1e-1)
learn.fit_flat_cos(10, 1e-1)
learn.fit_flat_cos(10, 1e-1)
learn.save('round2_9948')<jupyter_output><empty_output><jupyter_text>99,48%  TabNet<jupyter_code>emb_szs = get_emb_sz(to); print(emb_szs)
model = TabNetModel(emb_szs, len(to.cont_names), 10, n_d=16, n_a=8, n_steps=1); 
opt_func = partial(Adam, wd=0.01, eps=1e-5)
learn = Learner(dls, model, loss_func=CrossEntropyLossFlat(), opt_func=ranger, lr=3e-2, metrics=[accuracy])
learn.lr_find()
learn.fit_flat_cos(10, 5e-1)
learn.fit_flat_cos(10, 3e-1)
learn.fit_flat_cos(10, 1e-1)
learn.fit_flat_cos(10, 1e-1)
learn.fit_flat_cos(10, 1e-1)
learn.fit_flat_cos(10, 1e-1)
learn.fit_flat_cos(10, 1e-1)
learn.fit_flat_cos(10, 1e-1)
learn.save('best_97')
learn.load('best_97')
learn.fit_flat_cos(10, 1e-1)
learn.fit_flat_cos(10, 1e-1)
learn.fit_flat_cos(10, 1e-2)
learn.fit_flat_cos(10, 1e-1)
learn.fit_flat_cos(10, 1e-1)
learn.fit_flat_cos(10, 5e-2)
learn.fit_one_cycle(10, 5e-2)
learn.fit_one_cycle(10, 5e-2)
learn.fit_one_cycle(10, 5e-2)
learn.fit_one_cycle(10, 5e-2)
learn.fit_one_cycle(10, 5e-2)
learn.fit_one_cycle(10, 5e-2)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.16419981498612396, 'fraction_numerical': 0.09851988899167438, 'mean_word_length': 6.891727493917275, 'pattern_counts': {'":': 0, '<': 29, '<?xml version=': 0, '>': 29, 'https://': 3, 'lorem ipsum': 0, 'www.': 1, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '7', 'dir': 'jupyter-structured-clean-dedup', 'id': '590966', 'license': 'Apache-2.0', 'n_tokens_mistral': 3479, 'n_tokens_neox': 3162, 'n_words': 456, 'path': 'external_nbs/poker_hand_induction_fastai_x_tabnet_final.ipynb', 'repo_name': 'cwerner/tabnet-exp', 'size': '6486'}