<jupyter_start><jupyter_text>Self-Supervision Examples (Language)  Imports and Helpers<jupyter_code>from functools import partial

from torchtext.datasets import IMDB
from torchtext.data import Field, LabelField, BucketIterator

from transformers import BertTokenizer

from collators import MLMCollator, SequentialMLMCollator, PermutationMLMCollator, \
                      TextInfillingCollator, TokenDeletionCollator, DocumentRotationCollator, \
                      PermutationCollator
TOKENIZER = BertTokenizer.from_pretrained('bert-base-uncased',
                                          do_lower_case=True,
                                          do_basic_tokenize=True,
                                          unk_token='[UNK]',
                                          sep_token='[SEP]',
                                          pad_token='[PAD]',
                                          cls_token='[CLS]',
                                          mask_token='[MASK]')

encode = partial(TOKENIZER.encode, max_length=48)
pad_token_id = TOKENIZER.convert_tokens_to_ids(TOKENIZER.pad_token)
# NOTE: We add the pad token ID as discussed in https://github.com/pytorch/text/issues/609
TEXT = Field(use_vocab=False, tokenize=encode, pad_token=pad_token_id, batch_first=True)
LABEL = LabelField()
train_dataset, test_dataset = IMDB.splits(text_field=TEXT, label_field=LABEL)
LABEL.build_vocab(train_dataset)
train_loader, test_loader = BucketIterator.splits(datasets=(train_dataset, test_dataset), batch_size=1)
def simplify_label(label):
    tokens = label.split()
    tokens = [token for token in tokens if token != '[UNK]']
    tokens = ['%d. %s' % (i + 1, token) for i, token in enumerate(tokens)]
    return ', '.join(tokens)<jupyter_output><empty_output><jupyter_text>Masked Language Modeling (MLM) [BERT]<jupyter_code>collator = MLMCollator(tokenizer=TOKENIZER, mask_probability=0.25)

for batch in train_loader:
    x, _ = batch.text, batch.label
    examples, labels = collator(x)
    print('Input:\n------\n%s \n\nOutput:\n-------\n%s' % 
          (TOKENIZER.decode(examples[0].tolist()), simplify_label(TOKENIZER.decode(labels[0].tolist()))))
    break<jupyter_output>Using bos_token, but it is not set yet.
Using eos_token, but it is not set yet.<jupyter_text>Sequential MLM<jupyter_code>collator = SequentialMLMCollator(tokenizer=TOKENIZER)

for batch in train_loader:
    x, _ = batch.text, batch.label
    examples, labels = collator(x)
    print('Input 1:\n--------\n%s \n\nOutput 1:\n---------\n%s' % 
          (TOKENIZER.decode(examples[0].tolist()), simplify_label(TOKENIZER.decode(labels[0].tolist()))))
    print('\n\n')
    print('Input 2:\n--------\n%s \n\nOutput 2:\n---------\n%s' % 
          (TOKENIZER.decode(examples[1].tolist()), simplify_label(TOKENIZER.decode(labels[1].tolist()))))
    print('\n\n')
    print('...')
    break<jupyter_output>Using bos_token, but it is not set yet.
Using eos_token, but it is not set yet.<jupyter_text>Permutation-based MLM [XLNET]<jupyter_code>collator = PermutationMLMCollator(tokenizer=TOKENIZER, mask_probability=0.15, max_span_length=5)

for batch in train_loader:
    x, _ = batch.text, batch.label
    examples, permutation_mask, labels = collator(x)
    print('Input:\n------\n%s \n\nOutput:\n-------\n%s' % 
          (TOKENIZER.decode(examples[0].tolist()), simplify_label(TOKENIZER.decode(labels[0].tolist()))))
    print('\n')
    
    print('Permutation Mask Shape: ', permutation_mask.shape)
    break<jupyter_output>Using bos_token, but it is not set yet.
Using eos_token, but it is not set yet.<jupyter_text>Text-Infilling MLM [SPANBERT]<jupyter_code>collator = TextInfillingCollator(tokenizer=TOKENIZER, mask_probability=0.15, 
                                 mean_span_length=3, mask_strategy='spanbert')

for batch in train_loader:
    x, _ = batch.text, batch.label
    examples, labels = collator(x)
    print('Input:\n------\n%s \n\nOutput:\n-------\n%s' % 
          (TOKENIZER.decode(examples[0].tolist()), simplify_label(TOKENIZER.decode(labels[0].tolist()))))
    break<jupyter_output>Using bos_token, but it is not set yet.
Using eos_token, but it is not set yet.<jupyter_text>Text-Infilling MLM [BART]<jupyter_code>collator = TextInfillingCollator(tokenizer=TOKENIZER, mask_probability=0.15, 
                                 mean_span_length=3, mask_strategy='bart')

for batch in train_loader:
    x, _ = batch.text, batch.label
    examples, labels = collator(x)
    print('Input:\n------\n%s \n\nOutput:\n-------\n%s' % 
          (TOKENIZER.decode(examples[0].tolist()), TOKENIZER.decode(labels[0].tolist())))
    break<jupyter_output>Using bos_token, but it is not set yet.
Using eos_token, but it is not set yet.<jupyter_text>Token Deletion [BART]<jupyter_code>collator = TokenDeletionCollator(tokenizer=TOKENIZER, mask_probability=0.15)

for batch in train_loader:
    x, _ = batch.text, batch.label
    examples, labels = collator(x)
    print('Input:\n------\n%s \n\nOutput:\n-------\n%s' % 
          (TOKENIZER.decode(examples[0].tolist()), TOKENIZER.decode(labels[0].tolist())))
    break<jupyter_output>Using bos_token, but it is not set yet.
Using eos_token, but it is not set yet.<jupyter_text>Document Rotation [BART]<jupyter_code>collator = DocumentRotationCollator(tokenizer=TOKENIZER)

for batch in train_loader:
    x, _ = batch.text, batch.label
    examples, labels = collator(x)
    print('Input:\n------\n%s \n\nOutput:\n-------\n%s' % 
          (TOKENIZER.decode(examples[0].tolist()), TOKENIZER.decode(labels[0].tolist())))
    break<jupyter_output>Using bos_token, but it is not set yet.
Using eos_token, but it is not set yet.<jupyter_text>Permutation [BART]<jupyter_code>collator = PermutationCollator(tokenizer=TOKENIZER, permutation_policy='random')

for batch in train_loader:
    x, _ = batch.text, batch.label
    examples, labels = collator(x)
    print('Input:\n------\n%s \n\nOutput:\n-------\n%s' % 
          (TOKENIZER.decode(examples[0].tolist()), TOKENIZER.decode(labels[0].tolist())))
    break<jupyter_output>Using bos_token, but it is not set yet.
Using eos_token, but it is not set yet.
{'fraction_non_alphanumeric': 0.1429269086765424, 'fraction_numerical': 0.0076509848608171905, 'mean_word_length': 3.293501048218029, 'pattern_counts': {'":': 0, '<': 29, '<?xml version=': 0, '>': 29, 'https://': 1, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 16, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '9', 'dir': 'jupyter-structured-clean-dedup', 'id': '517525', 'license': 'MIT', 'n_tokens_mistral': 2106, 'n_tokens_neox': 2070, 'n_words': 393, 'path': 'language/examples.ipynb', 'repo_name': 'uzaymacar/self-supervision', 'size': '6143'}