<jupyter_start><jupyter_text>GPT-2 based letters generation<jupyter_code>%pip install gpt-2-simple
%pip install tensorflow==1.14
%pip install tensorflow-gpu==1.14.0<jupyter_output>Collecting gpt-2-simple
  Downloading https://files.pythonhosted.org/packages/6f/e4/a90add0c3328eed38a46c3ed137f2363b5d6a07bf13ee5d5d4d1e480b8c3/gpt_2_simple-0.7.1.tar.gz
Requirement already satisfied: regex in /usr/local/lib/python3.6/dist-packages (from gpt-2-simple) (2019.12.20)
Requirement already satisfied: requests in /usr/local/lib/python3.6/dist-packages (from gpt-2-simple) (2.23.0)
Requirement already satisfied: tqdm in /usr/local/lib/python3.6/dist-packages (from gpt-2-simple) (4.41.1)
Requirement already satisfied: numpy in /usr/local/lib/python3.6/dist-packages (from gpt-2-simple) (1.19.5)
Collecting toposort
  Downloading https://files.pythonhosted.org/packages/f2/7d/55784e894ee0cde2474fb977ffd1651e74e840a9f92e1d847f7e3115d5ec/toposort-1.6-py2.py3-none-any.whl
Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.6/dist-packages (from requests->gpt-2-simple) (2020.12.5)
Requirement already satisfied: chardet<4,>=3.0.2 in /usr/local/lib/python3.6/dist-[...]<jupyter_text>Download GPT-2 model<jupyter_code>import os
import sys
import requests
from tqdm import tqdm

def download_model(model="124M"):
  """Downloads a specific model.
    See https://github.com/minimaxir/gpt-2-simple#usage.
  """
  subdir = os.path.join('models', model)
  if not os.path.exists(subdir):
      os.makedirs(subdir)

  for filename in ['checkpoint','encoder.json','hparams.json','model.ckpt.data-00000-of-00001', 'model.ckpt.index', 'model.ckpt.meta', 'vocab.bpe']:
      r = requests.get("https://openaipublic.blob.core.windows.net/gpt-2/" + subdir + "/" + filename, stream=True)
      with open(os.path.join(subdir, filename), 'wb') as f:
          file_size = int(r.headers["content-length"])
          chunk_size = 1000
          with tqdm(ncols=100, desc="Fetching " + filename, total=file_size, unit_scale=True) as pbar:
              for chunk in r.iter_content(chunk_size=chunk_size):
                  f.write(chunk)
                  pbar.update(chunk_size)<jupyter_output><empty_output><jupyter_text>Train model<jupyter_code>import gpt_2_simple as gpt2
import os
import requests

model_name = "124M"
download_model(model=model_name)

# provide file for training
file_name = "input_letters"

# Resets model on re-finetune
import tensorflow as tf
tf.reset_default_graph()

sess = gpt2.start_tf_sess()
gpt2.finetune(
    sess,
    file_name,
    model_name=model_name,
    steps=1000,
    print_every=25,
    sample_every=200,
    save_every=500
)<jupyter_output>/usr/local/lib/python3.6/dist-packages/tensorflow/python/framework/dtypes.py:516: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.
  _np_qint8 = np.dtype([("qint8", np.int8, 1)])
/usr/local/lib/python3.6/dist-packages/tensorflow/python/framework/dtypes.py:517: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.
  _np_quint8 = np.dtype([("quint8", np.uint8, 1)])
/usr/local/lib/python3.6/dist-packages/tensorflow/python/framework/dtypes.py:518: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.
  _np_qint16 = np.dtype([("qint16", np.int16, 1)])
/usr/local/lib/python3.6/dist-packages/tensorflow/python/framework/dtypes.py:519: FutureWarning: Passing (type, 1) or [...]<jupyter_text>Generate new letters using trained model<jupyter_code>generated = gpt2.generate(
    sess,
    length=250,
    temperature=0.7,
    prefix="To my one and only...",
    return_as_list=True,
    run_name='run1'
)

print(generated)
import json

prefixes = None
with open("generate_with_prefixes", "r") as prefixes_file:
  prefixes = json.loads(prefixes_file.read())

assert prefixes is not None

generated = []
for prefix in prefixes:
  g = gpt2.generate(
      sess,
      length=250,
      temperature=0.7,
      prefix=prefix,
      return_as_list=True,
      run_name='run1'
  )
  generated.append({prefix: g[0]})
  print('generated', generated[-1])

print('generation finished.')
with open("generated", "w") as f:
  json.dump(generated, f)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.13898836168307968, 'fraction_numerical': 0.057072515666965085, 'mean_word_length': 4.403869407496977, 'pattern_counts': {'":': 0, '<': 16, '<?xml version=': 0, '>': 18, 'https://': 4, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 6, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '4', 'dir': 'jupyter-structured-clean-dedup', 'id': '422381', 'license': 'MIT', 'n_tokens_mistral': 1770, 'n_tokens_neox': 1660, 'n_words': 330, 'path': 'gpt-2/collab_generation.ipynb', 'repo_name': 'jan25/saint-valentin', 'size': '4468'}