<jupyter_start><jupyter_text>Load and process data<jupyter_code>text_field = Field('text', train = True, label = False, ignore = False, ix = 5, cname = 'text')
label_field = Field('label', train = False, label = True, cname = 'label', ignore = False, ix = 4)
ignore_field = Field('ignore', train = False, label = False, cname = 'ignore', ignore = True)

fields = [text_field, label_field]
dataset = GeneralDataset(data_dir = '~/PhD/projects/tools/mlearn/tests/',
                         ftype = 'csv', fields = fields, train = 'garcia_stormfront_train.tsv', dev = None,
                         test = 'garcia_stormfront_test.tsv', train_labels = None, tokenizer = lambda x: x.split(),
                         lower = True, preprocessor = None, transformations = None,
                         label_processor = None, sep = '\t', name = 'Test')
dataset.load('train')
dataset.load('test')<jupyter_output>Loading Test (train): 1914it [00:00, 39755.65it/s]
Loading Test (test): 478it [00:00, 11693.38it/s]<jupyter_text>Encode the documents and labels<jupyter_code>dataset.build_token_vocab(dataset.data)
dataset.build_label_vocab(dataset.data)
train = process_and_batch(dataset, dataset.data, 32, onehot = True)
test = process_and_batch(dataset, dataset.test, 32, onehot = True)
print("Number of batches:", len(train))
print("Length of first batch:", len(train[0]))<jupyter_output>Number of batches: 60
Length of first batch: 32<jupyter_text>Prepare the models  RNN<jupyter_code>model = oh.RNNClassifier(dataset.vocab_size(), hidden_dim = 128, embedding_dim = dataset.vocab_size(), output_dim = 3, batch_first = True)
optimizer = optim.Adam(model.parameters(), lr = 0.01)
loss = nn.NLLLoss()
with tqdm(train) as loop:
    for X, y in loop:
        res = model(X.long())
        l = loss(res, y)
        
        acc = accuracy_score(res.argmax(dim=1).cpu(), y.cpu())

        l.backward()
        optimizer.step()
        loop.set_postfix(loss = l.data.item() / X.shape[0], accuracy = acc)<jupyter_output>100%|██████████| 60/60 [02:10<00:00,  2.17s/it, accuracy=0.423, loss=0.125]<jupyter_text>LSTM<jupyter_code>model = oh.LSTMClassifier(dataset.vocab_size(), hidden_dim = 128, embedding_dim = 128, output_dim = 3, num_layers = 1, batch_first = True)
optimizer = optim.Adam(model.parameters(), lr = 0.01)
loss = nn.NLLLoss()
with tqdm(train) as loop:
    for X, y in loop:
        res = model(X.long())
        l = loss(res, y)
        
        acc = accuracy_score(res.argmax(dim=1).cpu(), y.cpu())

        l.backward()
        optimizer.step()
        loop.set_postfix(loss = l.data.item() / X.shape[0], accuracy = acc)<jupyter_output>0%|          | 0/60 [00:00<?, ?it/s]<jupyter_text>MLP<jupyter_code>model = oh.MLPClassifier(dataset.vocab_size(), hidden_dim = 128, embedding_dim = 128, output_dim = 3, batch_first = True)
optimizer = optim.Adam(model.parameters(), lr = 0.01)
loss = nn.NLLLoss()
with tqdm(train) as loop:
    for X, y in loop:
        res = model(X.long())
        l = loss(res, y)
        
        acc = accuracy_score(res.argmax(dim=1).cpu(), y.cpu())

        l.backward()
        optimizer.step()
        loop.set_postfix(loss = l.data.item() / X.shape[0], accuracy = acc)<jupyter_output><empty_output><jupyter_text>CNN<jupyter_code>model = oh.CNNClassifier(window_sizes = [2,3,4], num_filters = 128, max_feats = dataset.vocab_size(), 
                      output_dim = 3, vocab_size = dataset.vocab_size(), hidden_dim = 128,
                      batch_first = True)
optimizer = optim.Adam(model.parameters(), lr = 0.01)
loss = nn.NLLLoss()
with tqdm(train) as loop:
    for X, y in loop:
        res = model(X.long())
        l = loss(res, y)
        
        acc = accuracy_score(res.argmax(dim=1).cpu(), y.cpu())

        l.backward()
        optimizer.step()
        loop.set_postfix(loss = l.data.item() / X.shape[0], accuracy = acc)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.15198973042362002, 'fraction_numerical': 0.031578947368421054, 'mean_word_length': 3.3579418344519016, 'pattern_counts': {'":': 0, '<': 23, '<?xml version=': 0, '>': 21, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '6', 'dir': 'jupyter-structured-clean-dedup', 'id': '562222', 'license': 'MIT', 'n_tokens_mistral': 1487, 'n_tokens_neox': 1368, 'n_words': 292, 'path': 'notebooks/Onehot Tensor Models.ipynb', 'repo_name': 'ZeerakW/mlearn', 'size': '3895'}