<jupyter_start><jupyter_text>Load<jupyter_code>from torch.nn import TransformerEncoder, TransformerEncoderLayer
class SeqHead(nn.Module):
    def __init__(self):
        super().__init__()
#         d_model = 2048+6+1
        d_model = 2048
        n_head = 4
#         self.linear = nn.Linear(d_model+7, d_model)
        encoder_layers = TransformerEncoderLayer(d_model, n_head, d_model*2)
        self.transformer = TransformerEncoder(encoder_layers, 4)
        
        self.head = nn.Sequential(nn.Linear(d_model,6))
        
    def forward(self, x):
        x,x_p,x_m = x
#         x = torch.cat(x, axis=-1)
#         x = self.linear(x)
        feat = self.transformer(x.transpose(0,1))
        return self.head(feat.transpose(0,1))
m = SeqHead().cuda()
name = 'seutao_transformer'
learn = get_learner(dls, m)
learn.add_cb(DePadLoss())
learn.lr_find()<jupyter_output><empty_output><jupyter_text>Train<jupyter_code>do_fit(learn, 6, 1e-4)
learn.save(f'runs/{name}-1')<jupyter_output><empty_output><jupyter_text>Submit<jupyter_code>learn.dls = get_seutao_dls_meta(Meta.df_tst, np_file_test, csv_file_test, bs=1, test=True)
learn.load(f'runs/{name}-1')
preds,targs = learn.get_preds()
# df_series = Meta.df_tst.sort_values(['SeriesInstanceUID', "ImagePositionPatient2"])
sub_fn = f'subm/{name}'
pred_csv = submission(Meta.df_tst, preds, fn=sub_fn)
api.competition_submit(f'{sub_fn}.csv', sub_fn, 'rsna-intracranial-hemorrhage-detection')
api.competitions_submissions_list('rsna-intracranial-hemorrhage-detection')[0]
xb,yb = dls_test.one_batch()
submit_predictions(m, f'runs/{name}-1', f'subm/{name}', 'testing seutao features - transformer', dfunc=get_seutao_dls_meta)
api.competitions_submissions_list('rsna-intracranial-hemorrhage-detection')[0]<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.14382022471910114, 'fraction_numerical': 0.016853932584269662, 'mean_word_length': 5.078498293515358, 'pattern_counts': {'":': 0, '<': 13, '<?xml version=': 0, '>': 13, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '3', 'dir': 'jupyter-structured-clean-dedup', 'id': '347590', 'license': 'Apache-2.0', 'n_tokens_mistral': 720, 'n_tokens_neox': 694, 'n_words': 86, 'path': '06_seutao_features/06_seutao_features_04_Transformer.ipynb', 'repo_name': 'bearpelican/rsna_retro', 'size': '1780'}