<jupyter_start><jupyter_text>Dependencies, embeddings, transformers<jupyter_code>! git clone https://github.com/josipjukic/Adversarial-NLP.git --quiet
% cd /content/Adversarial-NLP/src

% mkdir .vector_cache
% cp '/content/drive/My Drive/Master Thesis/glove/glove.6B.100d.txt.pt' .vector_cache/

! pip install transformers --quiet<jupyter_output><empty_output><jupyter_text>Experiments<jupyter_code>import torch
import torch.nn as nn
import torch.optim as optim
from torchtext import data
from torchtext import datasets
import spacy
from data_utils import (load_data, set_seed_everywhere)
from training import run_experiment
from metrics import init_tqdms
SEED = 42
set_seed_everywhere(SEED)

LOAD_PATH = '/content/drive/My Drive/Master Thesis/AG'
MAX_VOCAB_SIZE = 25_000
EMBEDDINGS_FILE = 'glove.6B.100d'

splits, fields = load_data(LOAD_PATH,
                           MAX_VOCAB_SIZE=MAX_VOCAB_SIZE,
                           EMBEDDINGS_FILE=EMBEDDINGS_FILE,
                           float_label=False)
TEXT, LABEL, *_ = fields
from argparse import Namespace
from data_utils import expand_paths
from embeddings import get_embeddings
from models import PackedRNN

dataset_name = 'AG'
rnn_type = 'LSTM'
bidirectional = True
identifier = ('bi' if bidirectional else '') + rnn_type
args = Namespace(
    # Data and Path hyper parameters
    model_save_file=f'{identifier}.torch',
    train_state_file=f'stats_{identifier}.json',
    save_dir=f'/content/drive/My Drive/Master Thesis/torch_models/{dataset_name}',
    PAD_IDX = TEXT.vocab.stoi[TEXT.pad_token],
    UNK_IDX = TEXT.vocab.stoi[TEXT.unk_token],
    # Model hyper parameters
    input_dim = len(TEXT.vocab),
    embedding_dim=100,
    hidden_dim=256,
    output_dim = len(LABEL.vocab) if len(LABEL.vocab) > 2 else 1,
    num_layers=2,
    bidirectional=bidirectional,
    rnn_type=rnn_type,
    # Training hyper parameter
    seed=SEED,
    learning_rate=0.001,
    dropout_p=0.5,
    batch_size=64,
    num_epochs=20,
    early_stopping_criteria=5,
    # Runtime option
    device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
)

expand_paths(args)

model = PackedRNN(
    args.embedding_dim, 
    args.hidden_dim, 
    args.output_dim, 
    args.num_layers,
    get_embeddings(TEXT, args),
    args.bidirectional,
    args.dropout_p, 
    args.PAD_IDX,
    args.rnn_type,
    args.device
)

model = model.to(args.device)

train_iterator, valid_iterator, test_iterator = data.BucketIterator.splits(
    splits, 
    batch_size=args.batch_size,
    sort_within_batch=True,
    sort_key = lambda x: len(x.text),
    device=args.device)
iterator = dict(train=train_iterator, valid=valid_iterator, test=test_iterator)
criterion = nn.CrossEntropyLoss()
optimizer = optim.Adam(model.parameters(), lr=args.learning_rate)
run_experiment(args, model, iterator, optimizer, criterion, init_tqdms(args, iterator))<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.09893507385778083, 'fraction_numerical': 0.01202335967021642, 'mean_word_length': 4.228007181328546, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 10, 'https://': 1, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '2', 'dir': 'jupyter-structured-clean-dedup', 'id': '414955', 'license': 'MIT', 'n_tokens_mistral': 1062, 'n_tokens_neox': 1022, 'n_words': 194, 'path': 'src/experiments.ipynb', 'repo_name': 'josipjukic/Adversarial-NLP', 'size': '2911'}