<jupyter_start><jupyter_text>`REMEMBER` Ctrl+Shift+P in notebooks to bring up finder  For installation in collab (restart runtime)```py!pip install gputil!pip install torch==1.1.0!pip install spacy-transformers[cuda100]==0.5.1!python -m spacy download en_trf_xlnetbasecased_lg!python -m spacy download en_trf_bertbaseuncased_lg```<jupyter_code>import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
import spacy
import torch

torch.cuda.device(0)
print(torch.cuda.get_device_name(0))
print(f'pytorch {torch.__version__}, spacy {spacy.__version__}')

%matplotlib inline
nlp = spacy.load('en_trf_distilbertbaseuncased_lg')
doc = nlp("Apple shares rose on the news. Apple pie is delicious.")
print(doc[0].similarity(doc[7]))
print(doc._.trf_last_hidden_state.shape)<jupyter_output>0.77565545
(16, 768)<jupyter_text>Transfer learningThe main use case for pretrained transformer models is transfer learning. You load in a large generic model pretrained on lots of text, and start training on your smaller dataset with labels specific to your problem.<jupyter_code>import spacy
import GPUtil
import torch
import numpy
from numpy.testing import assert_almost_equal
from scipy.spatial import distance
import cupy
import numpy as np
is_using_gpu = spacy.prefer_gpu()
if is_using_gpu:
    print("Using GPU!")
    torch.set_default_tensor_type("torch.cuda.FloatTensor")
    print("GPU Usage")
    GPUtil.showUtilization()<jupyter_output>Using GPU!
GPU Usage
| ID | GPU | MEM |
------------------
|  0 |  3% | 63% |<jupyter_text>BERT uses wordpieces (e.g. playing -> play + ing) instead of words. This is effective in reducing the size of the vocabulary and increases the amount of data that is available for each word.<jupyter_code>nlp = spacy.load("en_trf_distilbertbaseuncased_lg")
doc = nlp("Here is some text to encode.")

assert doc.tensor.shape == (7, 768)  # Always has one row per token
print(doc._.trf_word_pieces_)        # String values of the wordpieces<jupyter_output>['[CLS]', 'here', 'is', 'some', 'text', 'to', 'en', '##code', '.', '[SEP]']<jupyter_text>The raw transformer output has one row per wordpiece.<jupyter_code>print(doc._.trf_word_pieces)  # Wordpiece IDs (note: *not* spaCy's hash values!)
print(doc._.trf_alignment)    # Alignment between spaCy tokens and wordpieces<jupyter_output>[101, 2182, 2003, 2070, 3793, 2000, 4372, 16044, 1012, 102]
[[1], [2], [3], [4], [5], [6, 7], [8]]<jupyter_text>Build a Sentiment Classifier using spaCy Transformers<jupyter_code>import thinc
import random
import spacy
import GPUtil
import torch
from spacy.util import minibatch
from tqdm.auto import tqdm
import unicodedata
import wasabi
import numpy
from collections import Counter
spacy.util.fix_random_seed(0)
is_using_gpu = spacy.prefer_gpu()
if is_using_gpu:
    torch.set_default_tensor_type("torch.cuda.FloatTensor")
    print("GPU Usage")
    GPUtil.showUtilization()<jupyter_output>GPU Usage
| ID | GPU | MEM |
------------------
|  0 |  6% | 13% |<jupyter_text>Load IMDB movie database for sentiment analysis<jupyter_code>def _prepare_partition(text_label_tuples, *, preprocess=False):
    texts, labels = zip(*text_label_tuples)
    cats = [{"POSITIVE": bool(y), "NEGATIVE": not bool(y)} for y in labels]
    return texts, cats

def load_data(*, limit=0, dev_size=2000):
    """Load data from the IMDB dataset, splitting off a held-out set."""
    if limit != 0:
        limit += dev_size
    assert dev_size != 0
    train_data, _ = thinc.extra.datasets.imdb(limit=limit)
    assert len(train_data) > dev_size
    random.shuffle(train_data)
    dev_data = train_data[:dev_size]
    train_data = train_data[dev_size:]
    train_texts, train_labels = _prepare_partition(train_data, preprocess=False)
    dev_texts, dev_labels = _prepare_partition(dev_data, preprocess=False)
    return (train_texts, train_labels), (dev_texts, dev_labels)
(train_texts, train_cats), (eval_texts, eval_cats) = load_data()
GPUtil.showUtilization()<jupyter_output>| ID | GPU | MEM |
------------------
|  0 |  0% | 13% |<jupyter_text>`Models`  en_trf_distilbertbaseuncased_lg  en_trf_xlnetbasecased_lg  en_trf_bertbaseuncased_lg<jupyter_code># !python -m spacy download en_trf_xlnetbasecased_lg
## TODO Try with other model
model_choice = "en_trf_distilbertbaseuncased_lg" 

# model_choice = "en_trf_xlnetbasecased_lg"
nlp = spacy.load(model_choice)
print(nlp.pipe_names)
textcat = nlp.create_pipe("trf_textcat", config={"architecture": "softmax_class_vector"})
print(f"Loaded model '{model_choice}'")
# add label to text classifier
textcat.add_label("POSITIVE")
textcat.add_label("NEGATIVE")
print("Labels:", textcat.labels)
nlp.add_pipe(textcat, last=True)
print(f"Using {len(train_texts)} training docs, {len(eval_texts)} evaluation")
# total_words = sum(len(text.split()) for text in train_texts)
train_data = list(zip(train_texts, [{"cats": cats} for cats in train_cats]))
def cyclic_triangular_rate(min_lr, max_lr, period):
    it = 1
    while True:
        # https://towardsdatascience.com/adaptive-and-cyclical-learning-rates-using-pytorch-2bf904d18dee
        cycle = numpy.floor(1 + it / (2 * period))
        x = numpy.abs(it / period - 2 * cycle + 1)
        relative = max(0, 1 - x)
        yield min_lr + (max_lr - min_lr) * relative
        it += 1
def evaluate(nlp, texts, cats, pos_label):
    tp = 0.0  # True positives
    fp = 0.0  # False positives
    fn = 0.0  # False negatives
    tn = 0.0  # True negatives
    total_words = sum(len(text.split()) for text in texts)
    with tqdm(total=total_words, leave=False) as pbar:
        for i, doc in enumerate(nlp.pipe(texts, batch_size=batch_size)):
            gold = cats[i]
            for label, score in doc.cats.items():
                if label not in gold:
                    continue
                if label != pos_label:
                    continue
                if score >= 0.5 and gold[label] >= 0.5:
                    tp += 1.0
                elif score >= 0.5 and gold[label] < 0.5:
                    fp += 1.0
                elif score < 0.5 and gold[label] < 0.5:
                    tn += 1
                elif score < 0.5 and gold[label] >= 0.5:
                    fn += 1
            pbar.update(len(doc.text.split()))
    precision = tp / (tp + fp + 1e-8)
    recall = tp / (tp + fn + 1e-8)
    if (precision + recall) == 0:
        f_score = 0.0
    else:
        f_score = 2 * (precision * recall) / (precision + recall)
    return {"textcat_p": precision, "textcat_r": recall, "textcat_f": f_score}

n_iter=4
n_texts=25     # Changed number of texts from 1000 to 75 to relieve pressue on GPU memory
batch_size=1   # batch-szie changed from 8 to 4 to relieve pressure on GPU memory
learn_rate=2e-5
max_wpb=1000
pos_label="POSITIVE"


# Initialize the TextCategorizer, and create an optimizer.
optimizer = nlp.resume_training()
optimizer.alpha = 0.001
optimizer.trf_weight_decay = 0.005
optimizer.L2 = 0.0
learn_rates = cyclic_triangular_rate(
    learn_rate / 3, learn_rate * 3, 2 * len(train_data) // batch_size
    )
print("Training the model...")
print("{:^5}\t{:^5}\t{:^5}\t{:^5}".format("LOSS", "P", "R", "F"))

pbar = tqdm(total=100, leave=False)
results = []
epoch = 0
step = 0
eval_every = 100
patience = 3
while True:
    # Train and evaluate
    losses = Counter()
    random.shuffle(train_data)
    batches = minibatch(train_data, size=batch_size)
    for batch in batches:
        optimizer.trf_lr = next(learn_rates)
        texts, annotations = zip(*batch)
        nlp.update(texts, annotations, sgd=optimizer, drop=0.1, losses=losses)
        pbar.update(1)
        if step and (step % eval_every) == 0:
            pbar.close()
            with nlp.use_params(optimizer.averages):
                scores = evaluate(nlp, eval_texts, eval_cats, pos_label)
            results.append((scores["textcat_f"], step, epoch))
            print(
                "{0:.3f}\t{1:.3f}\t{2:.3f}\t{3:.3f}".format(
                    losses["trf_textcat"],
                    scores["textcat_p"],
                    scores["textcat_r"],
                    scores["textcat_f"],
                )
            )
            pbar = tqdm(total=eval_every, leave=False)
        step += 1
    epoch += 1
    print(f"epoch {epoch}")
    # Stop if no improvement in HP.patience checkpoints
    if results:
        best_score, best_step, best_epoch = max(results)
        print(f"best score: {best_score}  best_step : {best_step}  best epoch : {best_epoch} ")
        print(f"break clause: {((step - best_step) // eval_every)}")
        if ((step - best_step) // eval_every) >= patience:
            break

    msg = wasabi.Printer()
    table_widths = [2, 4, 6]
    msg.info(f"Best scoring checkpoints")
    msg.row(["Epoch", "Step", "Score"], widths=table_widths)
    msg.row(["-" * width for width in table_widths])
    for score, step, epoch in sorted(results, reverse=True)[:10]:
        msg.row([epoch, step, "%.2f" % (score * 100)], widths=table_widths)

    # Test the trained model
    test_text = eval_texts[0]
    doc = nlp(test_text)
    print(test_text, doc.cats)<jupyter_output>Training the model...
LOSS 	  P  	  R  	  F
{'fraction_non_alphanumeric': 0.12582056892778992, 'fraction_numerical': 0.02396061269146608, 'mean_word_length': 3.34458174904943, 'pattern_counts': {'":': 7, '<': 26, '<?xml version=': 0, '>': 29, 'https://': 1, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 3, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '7', 'dir': 'jupyter-structured-clean-dedup', 'id': '271042', 'license': 'MIT', 'n_tokens_mistral': 3345, 'n_tokens_neox': 3135, 'n_words': 800, 'path': '01DistilbertTransformer.ipynb', 'repo_name': 'lukexyz/Lyrical-Genius', 'size': '9140'}