<jupyter_start><jupyter_text>Contents1&nbsp;&nbsp;Compressive Transformer1.1&nbsp;&nbsp;Options:1.2&nbsp;&nbsp;setup1.3&nbsp;&nbsp;Create Model1.4&nbsp;&nbsp;Model summary1.5&nbsp;&nbsp;Create train/test data-generators1.6&nbsp;&nbsp;Create addtional Run Configurations1.7&nbsp;&nbsp;Train<jupyter_code>import os
import re
import sys
import numpy as np
import pandas as pd

from time import sleep, time
from keras import backend as K
from keras.models import Model 
from keras.models import Sequential as SequentialModel
from keras.models import load_model
from keras.layers import Dense, Conv1D, LSTM, Dropout, Embedding, Layer, Input, Flatten, concatenate as Concatenate, Lambda, Add
from keras.callbacks import Callback
from keras.utils import to_categorical
from keras.preprocessing.text import Tokenizer as KerasTokenizer
from tqdm.notebook import tqdm
from sometimer import timer, time_this_method

%load_ext autoreload
%autoreload 2
%reload_ext autoreload

sys.path.insert(0, '../ct')

import load
import preprocess

from preprocess.tokenize import Tokenizer
from preprocess.tokenize import separator_samples

from model.layers import LayerNormalization
from model.layers import ContentBasedAttention_CT
from model.layers import ScaledDotProductAttention
from model.layers import MultiHeadAttention
from model.layers import content_based_attention

from model import CompressiveTransformer
from model import AttentionReconstruction

from model.callbacks import ClearCompressedMemory, WriteLogsToFile
from train.generators import next_token_batch_generator<jupyter_output>Using TensorFlow backend.<jupyter_text>Compressive Transformer  Options:<jupyter_code># Preprocessing
vocab_size=16384
lowercase=False
language='english'
train_val_split=0.8
input_paths={'en': '../data/wma-en-de/input/train-en.txt',
             'de': '../data/wma-en-de/input/train-de.txt'}
tokenizer_output_path=f'../data/wma-en-de/tokenizer/' \
                      f'en-de-v0-t{vocab_size}{"-lowercase" if lowercase else ""}.tok'
epochs=100
batch_size=128
d_layers=2
sequence_length=128
d_model=128
train_steps=1200000
validation_steps=10000<jupyter_output><empty_output><jupyter_text>setup<jupyter_code>wma = load.wma(input_paths['en'], 
               input_paths['de'])
tokenizer = Tokenizer(input_paths=list(input_paths.values()),
                      tokenizer_output_path=tokenizer_output_path,
                      vocab_size=vocab_size,
                      lowercase=lowercase)
_, _, train, val = preprocess.wma(input_paths,
                                  wma[[language]],
                                  tokenizer_output_path,
                                  train_val_split=train_val_split,
                                  vocab_size=vocab_size,
                                  language=language,
                                  lowercase=lowercase,
                                  tokenizer=tokenizer)
epoch_steps=train_steps or len(train)
validation_steps=validation_steps or len(val)
steps_per_epoch=epoch_steps//sequence_length - 1
steps_per_validation=validation_steps//sequence_length - 1<jupyter_output><empty_output><jupyter_text>Create Model<jupyter_code>ct = CompressiveTransformer(d_layers=d_layers,
                            sequence_length=sequence_length, 
                            d_model=d_model,
                            memory_size=256,
                            compressed_memory_size=256,
                            d_k=16, 
                            d_heads=2, 
                            output_size=vocab_size,
                            batch_size=batch_size,
                            vocab_size=vocab_size)
ct.compile(optimizer='Adam',
           loss='categorical_crossentropy',
           metrics=['accuracy']
          )<jupyter_output><empty_output><jupyter_text>Model summary<jupyter_code>ct.summary()<jupyter_output>Model: "CompressiveTransformer"
__________________________________________________________________________________________________
Layer (type)                    Output Shape         Param #     Connected to                     
==================================================================================================
x (InputLayer)                  (None, 128)          0                                            
__________________________________________________________________________________________________
h_L0 (Embedding)                (None, 128, 128)     2097152     x[0][0]                          
__________________________________________________________________________________________________
compressed_memory (InputLayer)  (None, 2, 256, 128)  0                                            
__________________________________________________________________________________________________
memory (InputLayer)             (None, 2, 256, 128)  0                       [...]<jupyter_text>Create train/test data-generators<jupyter_code>generator = next_token_batch_generator(ct=ct,
                                       data=train,
                                       data_path=None,
                                       epoch_steps=epoch_steps, 
                                       sequence_length=sequence_length, 
                                       batch_size=sequence_length,
                                       vocab_size=vocab_size)
validation_generator = next_token_batch_generator(ct=ct, 
                                                  data=val,
                                                  data_path=None,
                                                  epoch_steps=validation_steps, 
                                                  sequence_length=sequence_length, 
                                                  batch_size=sequence_length,
                                                  vocab_size=vocab_size)<jupyter_output><empty_output><jupyter_text>Create addtional Run Configurations<jupyter_code>callbacks = [ClearCompressedMemory(),
             WriteLogsToFile(filepath='training-logs/ct.txt', overwrite_old_file=False)]<jupyter_output><empty_output><jupyter_text>Train<jupyter_code>ct.fit_generator(generator(), 
                 steps_per_epoch=steps_per_epoch,
                 epochs=epochs,
                 callbacks=callbacks,
                 validation_data=validation_generator(),
                 validation_steps=steps_per_validation,
                 shuffle=False
                 )<jupyter_output>/home/vs/anaconda3/envs/keras/lib/python3.7/site-packages/tensorflow_core/python/framework/indexed_slices.py:433: UserWarning: Converting sparse IndexedSlices to a dense Tensor of unknown shape. This may consume a large amount of memory.
  "Converting sparse IndexedSlices to a dense Tensor of unknown shape. "
{'fraction_non_alphanumeric': 0.08257151282807432, 'fraction_numerical': 0.015334709525213801, 'mean_word_length': 2.3763066202090593, 'pattern_counts': {'":': 0, '<': 30, '<?xml version=': 0, '>': 30, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 29, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '8', 'dir': 'jupyter-structured-clean-dedup', 'id': '545516', 'license': 'MIT', 'n_tokens_mistral': 1816, 'n_tokens_neox': 1709, 'n_words': 306, 'path': 'notebooks/Test Compressive Transformer.ipynb', 'repo_name': 'ViktorStagge/CompressiveTransformer', 'size': '6782'}