<jupyter_start><jupyter_text><jupyter_code>!pip install gputil
!pip install spacy-pytorch-transformers[cuda100]
!python -m spacy download en_pytt_robertabase_lg<jupyter_output>Collecting gputil
  Downloading https://files.pythonhosted.org/packages/ed/0e/5c61eedde9f6c87713e89d794f01e378cfd9565847d4576fa627d758c554/GPUtil-1.4.0.tar.gz
Building wheels for collected packages: gputil
  Building wheel for gputil (setup.py) ... [?25l[?25hdone
  Created wheel for gputil: filename=GPUtil-1.4.0-cp36-none-any.whl size=7410 sha256=3191b17a5642ad2df8f34e0222601483961f003eb7dd18d86d658fcc2b1681d9
  Stored in directory: /root/.cache/pip/wheels/3d/77/07/80562de4bb0786e5ea186911a2c831fdd0018bda69beab71fd
Successfully built gputil
Installing collected packages: gputil
Successfully installed gputil-1.4.0
Collecting spacy-pytorch-transformers[cuda100]
[?25l  Downloading https://files.pythonhosted.org/packages/fd/46/3271586944ee5e0bd493df03b1ad189eb9ccdad1d2476aeb843b0d2f1b47/spacy_pytorch_transformers-0.4.0-py3-none-any.whl (62kB)
[K     |████████████████████████████████| 71kB 3.5MB/s 
[?25hRequirement already satisfied: srsly<1.1.0,>=0.0.7 in /usr/local/lib/python3.6/dist[...]<jupyter_text>RESTART runtime after download<jupyter_code>import spacy
import torch
import cupy
import numpy as np
import pandas as pd
import re
import matplotlib.pyplot as plt
import random
import time
import unicodedata
import GPUtil
import spacy_pytorch_transformers
import en_pytt_robertabase_lg

from spacy_pytorch_transformers import PyTT_Language, PyTT_WordPiecer, PyTT_TokenVectorEncoder, PyTT_TextCategorizer
from spacy_pytorch_transformers.util import cyclic_triangular_rate
from tqdm import tqdm
from spacy.util import minibatch
from numpy.testing import assert_almost_equal
from scipy.spatial import distance
from google.colab import drive
from wasabi import Printer
from sklearn.model_selection import train_test_split
from sklearn.preprocessing import OneHotEncoder as OHE
from sklearn.compose import make_column_transformer
from datetime import datetime as dt
from datetime import timezone as tz
from collections import defaultdict
drive.mount('/gdrive')
#%cd /gdrive/My\ Drive/data
base_path = '/gdrive/My Drive/data/'
is_using_gpu = spacy.prefer_gpu()
def check_gpu(is_using_gpu):
  if is_using_gpu:
    print("Using GPU!")
    torch.set_default_tensor_type("torch.cuda.FloatTensor")
    print("GPU Usage")
    GPUtil.showUtilization()

spacy.prefer_gpu()
csv_path = base_path + 'data_dadjokes.csv'

dj = pd.read_csv(csv_path, na_values=["[deleted]", None,'N/A', ' ', '[removed]', ''], index_col=None, sep='|')
print(dj.isna().sum())
print(dj[dj.title.isna()].isna().sum())
dj.drop(index = dj.index[dj.title.isna()], inplace=True)

dj.isna().sum()
plt.scatter(dj.score, dj.num_comments);
plt.xlabel('Score')
plt.ylabel('Total Comments')
plt.title('Score vs. Number of Comments on\nSubmissions in /r/dadjokes')
plt.show()
plt.xscale('log')
plt.yscale('log')
plt.xlabel('Total Comments')
plt.ylabel('Number of Submissions')
plt.title('Submissions by Number of Comments\nin /r/dadjokes')
plt.hist(dj.num_comments, bins=[0,1,3,10,50,dj.num_comments.max()]);
plt.show()
plt.xscale('log')
plt.yscale('log')
plt.xlabel('Score')
plt.ylabel('Number of Submissions')
plt.title('Submissions by Score (Upvotes) in /r/dadjokes')
plt.hist(dj.score, bins=[0,1,5,20,100,dj.score.max()]);
plt.show()
dj['is_crosspost'] = ~dj['parent_createdUTC'].isna()
dj.groupby(by='is_crosspost').mean()<jupyter_output><empty_output><jupyter_text>Crossposts receive significantly lower scores, num_comments than original posts in the dadjokes subreddit. This may be because viewers are easily redirected to the parent post to comments or upvote in other areas of reddit. However, the contents of the crossposted jokes (title and selftext) are pulled from the parent posts. Therefore crossposts will be analyzed separately when attempting to predict scores, num_comments.<jupyter_code>main_dj = dj[~dj.is_crosspost].drop(columns=['parent_createdUTC', 'is_crosspost'])
print(main_dj.shape)
train_dj, test_dj = train_test_split(main_dj, test_size=0.2)
print(f"Training sample size: {len(train_dj)}\nTesting sample size: {len(test_dj)}")

train_dj.reset_index(drop=True, inplace=True)
test_dj.reset_index(drop=True, inplace=True)
score_cats = [1, 5, 20, 100]
comment_cats = [1, 3, 10, 50]
newlines = re.compile(r'(\\n)|\s+]')

def _gettext(s):
  if isinstance(s, str):
    s = s.replace('&amp;#x200B;', '')
    s = s.replace('&amp;', 'and')
    s = newlines.sub(' ', s)
  return s


def prepare_text(data):
    data['title_clean'] = data.title.apply(_gettext)
    data['selftext_clean'] = data.selftext.apply(_gettext)
    data['score_cat'] = data.score.apply(lambda _: _categorize(_, score_cats))
    data['comment_cat'] = data.num_comments.apply(
        lambda _: _categorize(_, comment_cats))
    data['createdUTC'] = data.createdUTC.apply(
        lambda x: dt.fromtimestamp(x, tz.utc))
    return data


def _categorize(val, cutoffs):
    for level, upper_bound in enumerate(cutoffs):
        if val < upper_bound:
            return level + 1
    return len(cutoffs) + 1
train_dj = prepare_text(train_dj)
test_dj = prepare_text(test_dj)
train_dj.to_csv(base_path + 'train_dj.csv')
test_dj.to_csv(base_path + 'test_dj.csv')
nlp = en_pytt_robertabase_lg.load(torchscript=True)
textcat = nlp.create_pipe("pytt_textcat", config={"exclusive_classes": True,
                                                  'architecture': 'softmax_pooler_output',
                                                  "words_per_batch": 1000})
for label in range(1,6):
    textcat.add_label(label)
nlp.add_pipe(textcat, last=True)
def check_pipe_config(nlp):
  for p in nlp.pipe_names:
    print(repr(p))
    try:
      print(nlp.get_pipe(p).cfg)
    except:
      pass
    try:
      print('model component included?:', nlp.get_pipe(p).model, '\n')
    except:
      pass

check_pipe_config(nlp)
train_dj = pd.read_csv(base_path + 'train_dj.csv')
def complete_joke(title, selftext):
  if isinstance(selftext, str):
    return ' '.join([title, selftext])
  else:
    return title

train_texts = train_dj.apply(
    lambda row: complete_joke(row['title_clean'], row['selftext_clean']), axis=1)
#test_texts = test_dj.apply(
#    lambda row: complete_joke(row['title_clean'], row['selftext_clean']), axis=1)

TRAIN_DATA = list(zip(train_texts, [{"cats": {i: float(val==i) for i in range(1, 6)}} for val in train_dj['score_cat']]))
#TEST_DATA = list(zip(test_texts, [{"cats": {i: float(val==i) for i in range(1, 6)}} for val in test_dj['score_cat']]))
with open(base_path + 'TRAIN.txt', 'w') as train:
  for line in TRAIN_DATA:
    train.write(str(line) + '\n')
with open(base_path + 'TEST.txt', 'w') as test:
  for line in TEST_DATA:
    test.write(str(line) + '\n')
check_gpu(is_using_gpu)<jupyter_output>Using GPU!
GPU Usage
| ID | GPU | MEM |
------------------
|  0 |  0% | 12% |<jupyter_text>doc = nlp(train_texts[0], disable=['pytt_textcat'])print(doc)print([sent.text for sent in doc.sents])print(len([wp for wp in doc._.pytt_word_pieces_]), [wp for wp in doc._.pytt_word_pieces_])print(len(doc._.pytt_alignment), doc._.pytt_alignment)print(len(doc._.pytt_last_hidden_state), doc._.pytt_last_hidden_state)def spacy_alignment(doc):    print("After tokenization, this doc has {} tokens.".format(len(doc)))    print("The part-of-speech tags are:", [token.pos_ for token in doc])    if len(doc) < 10:        print("This is a pretty short document.")    return doc<jupyter_code>batch_size=5 #batch-szie changed to 4 to relieve pressure on GPU memory
learn_rate=2e-5
def _cross_entropy(yHat, y):
    if y == 1:
      return -log(yHat)
    else:
      return -log(1 - yHat)

def evaluate(nlp, texts, cats):
    cross_ent = 0
    correct = 0
    total_words = sum(len(text.split()) for text in texts)
    with tqdm(total=total_words, leave=False) as pbar:
        i = 0
        for doc in nlp.pipe(texts, batch_size=batch_size):
            gold = cats[i]
            for label, score in doc.cats.items():
                if label not in gold:
                    continue
                cross_ent += _cross_entropy(score, gold[label])
            if max(gold.keys(), key=(lambda x: gold[x])) == max(doc.cats.keys(), key=(lambda x: doc.cats[x])):
                correct += 1
            i += 1
            pbar.update(len(doc.text.split()))
    return correct / i * 100, cross_ent

is_using_gpu = spacy.prefer_gpu()
check_gpu(is_using_gpu)
!nvidia-smi
optimizer = nlp.resume_training()
optimizer.alpha = 0.001
optimizer.pytt_weight_decay = 0.005
optimizer.L2 = 0.0
learn_rates = cyclic_triangular_rate(
    learn_rate / 3, learn_rate * 3, 2 * len(TRAIN_DATA) // batch_size
    )
print("Training the model...")

pbar = tqdm(total=100, leave=False)
results = []
epoch = 0
step = 0
eval_every = 100
patience = 3
while True:
    # Train and evaluate
    losses = defaultdict()
    train_data, eval_data = train_test_split(TRAIN_DATA, test_size=0.2)
    batches = minibatch(train_data, size=batch_size)
    for batch in batches:
        optimizer.pytt_lr = next(learn_rates)
        texts, annotations = zip(*batch)
        nlp.update(texts, annotations, sgd=optimizer, drop=0.1, losses=losses)
        pbar.update(1)
        if step and (step % eval_every) == 0:
            pbar.close()
            with nlp.use_params(optimizer.averages):
                eval_texts, eval_annotations = zip(*eval_data)
                accuracy, cross_ent = evaluate(nlp, eval_texts, eval_annotations)
            results.append((cross_ent, accuracy, step, epoch))
            print("{:^5}\t{:^5}\t{:^5}".format("LOSS", "ACC", "CE"))
            print("{0:.3f}\t{1:.3f}\t{2:.3f}".format(
                losses["pytt_textcat"], accuracy, total_ent
                ))
            pbar = tqdm(total=eval_every, leave=False)
        step += 1
    epoch += 1
    print(f"epoch {epoch}")
    # Stop if no improvement in HP.patience checkpoints
    if results:
        best_score, acc, best_step, best_epoch = max(results)
        print(f"best score: {best_score}  best_step : {best_step}  best epoch : {best_epoch} ")
        print(f"break clause: {((step - best_step) // eval_every)}")
        if ((step - best_step) // eval_every) >= patience:
            break

    msg = Printer()
    table_widths = [2, 4, 6]
    msg.info(f"Best scoring checkpoints")
    msg.row(["Epoch", "Step", "Score"], widths=table_widths)
    msg.row(["-" * width for width in table_widths])
    for score, acc, step, epoch in sorted(results, reverse=True)[:10]:
        msg.row([epoch, step, "%.2f" % (score * 100)], widths=table_widths)

    # Test the trained model
    test_text = eval_texts[0]
    doc = nlp(test_text)
    print(test_text, doc.cats)

!nvidia-smi
!cat /proc/meminfo
!ps
a = []
a.append([1,2])
a
any((1,2,3) < 2)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1377785964265979, 'fraction_numerical': 0.030392337447043654, 'mean_word_length': 4.107714016933208, 'pattern_counts': {'":': 4, '<': 19, '<?xml version=': 0, '>': 17, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 8, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '4', 'dir': 'jupyter-structured-clean-dedup', 'id': '269874', 'license': 'MIT', 'n_tokens_mistral': 4191, 'n_tokens_neox': 3910, 'n_words': 843, 'path': 'pytorch_roberta.ipynb', 'repo_name': 'jojordan3/dad-joke-ai', 'size': '10858'}