<jupyter_start><jupyter_text>[View in Colaboratory](https://colab.research.google.com/github/delkind/paraphraser/blob/master/notebooks/paraphraser_bible.ipynb)<jupyter_code>%load_ext autoreload
%autoreload 2


# colab requirements
#!pip install spacy  #only for bible
!pip install pydrive   #to save to google-drive
!pip install num2words #only for numbers


import tensorflow as tf
import sys
import numpy as np

# our github proj!
!rm -r paraphraser  #remove previous github copy if needed
!git clone https://github.com/delkind/paraphraser.git
sys.path.append('paraphraser/src')


#usage example
from utils.persistency import Persistency   
from models import D_G_Model<jupyter_output>The autoreload extension is already loaded. To reload it, use:
  %reload_ext autoreload
Requirement already satisfied: pydrive in /usr/local/lib/python3.6/dist-packages (1.3.1)
Requirement already satisfied: google-api-python-client>=1.2 in /usr/local/lib/python3.6/dist-packages (from pydrive) (1.6.7)
Requirement already satisfied: oauth2client>=4.0.0 in /usr/local/lib/python3.6/dist-packages (from pydrive) (4.1.3)
Requirement already satisfied: PyYAML>=3.0 in /usr/local/lib/python3.6/dist-packages (from pydrive) (3.13)
Requirement already satisfied: six<2dev,>=1.6.1 in /usr/local/lib/python3.6/dist-packages (from google-api-python-client>=1.2->pydrive) (1.11.0)
Requirement already satisfied: httplib2<1dev,>=0.9.2 in /usr/local/lib/python3.6/dist-packages (from google-api-python-client>=1.2->pydrive) (0.11.3)
Requirement already satisfied: uritemplate<4dev,>=3.0.0 in /usr/local/lib/python3.6/dist-packages (from google-api-python-client>=1.2->pydrive) (3.0.0)
Requirement already satisfi[...]<jupyter_text>Dataset<jupyter_code>from dataset.bible import BibleDataset
dataset = BibleDataset(["asv", "ylt"], "https://raw.githubusercontent.com/scrollmapper/bible_databases/master/csv/t_",'.csv')

# example for datasets generators
def show_dataset_example():
  (x1,x2),y1=next(dataset.gen_g(dataset.train, batch_size=3))
  for b in range(len(x1)):
    print ('x1',x1.shape,dataset.recostruct_sentence(x1[b]))
    print ('x2',x2.shape,dataset.recostruct_sentence(x2[b]))
    print ('y1',y1.shape,dataset.recostruct_sentence(y1[b].argmax(axis=1)))
    print ('')  
  print ('results of gen_adv')  
  (x1, x2), (y1,y2) = next(dataset.gen_adv(dataset.train, batch_size=3, noise_std=0.5))  
  for b in range(len(x1)):
    print ('x1',x1.shape,dataset.recostruct_sentence(x1[b]))
    print ('x2',x2.shape,dataset.recostruct_sentence(x2[b]))
    print ('y1',y1.shape,dataset.recostruct_sentence(y1[b].argmax(axis=1)))
    print ('y2',y2[b])


show_dataset_example()  #unmark it to understand how the data looks

from models import D_G_Model,D_G_Trainer
from decoder import SamplingDecoder


model = D_G_Model(num_encoder_tokens=len(dataset.word2index),
                      num_decoder_tokens=len(dataset.word2index),  #from dataset 3628
                      style_out_size=len(dataset.style2index), #from dataset 2
                      cuddlstm=True,
                      latent_dim = 50, #twice the default. make it stronger! but slower
                      bidi_encoder = True,
                      adv_loss_weight=100,) #500
model.build_all()


sampler= SamplingDecoder(model)        

trainer = D_G_Trainer(model,dataset)

train_size = len(dataset.index2style) * (dataset.train[1] - dataset.train[0])
batch_size=64
epoc = int(train_size/batch_size)
print ('epoc is of',epoc,'of batches',batch_size,'total train_size',train_size)<jupyter_output>unoptimzied decode_sequence_batch, running each of the N sample seperatly
unoptimzied decode_sequence_batch, running each of the N sample seperatly
epoc is of 756 of batches 64 total train_size 48432<jupyter_text>cycle appraoch<jupyter_code>for outside_epoc in range(200):
  
  steps_d,steps_g=(20,20) #use 20 steps for verbosity
  for inside_epoc in range(20):
      #trainer.train_g(steps_d,batch_size=32,noise_std=0.0)
      trainer.train_d(steps_d,batch_size=32,noise=0.00)
      trainer.train_d_g(steps_g ,batch_size=32,noise=0.0,noise_std=1.0)
      trainer.train_g_cycle(steps_g,batch_size=32,noise_std=10.0)
      steps_d,steps_g= (100,100)
  trainer.plt_all()
  sampler.show_sample(dataset,'train'  ,sample_ids=[10,25],teacher_forcing=True)   #,8000+0<jupyter_output>Epoch 1/1
20/20 [==============================] - 0s 15ms/step - loss: 0.6935 - acc: 0.4969 - val_loss: 0.7006 - val_acc: 0.4062
Epoch 1/1
20/20 [==============================] - 2s 108ms/step - loss: 2.7788 - dcd_sfmax_loss: 2.7304 - styl_clsf_loss: 4.8376e-04 - val_loss: 3.9134 - val_dcd_sfmax_loss: 3.9063 - val_styl_clsf_loss: 7.0550e-05
Epoch 1/1
20/20 [==============================] - 8s 383ms/step - loss: 2.8980 - dcd_sfmax_loss: 2.8662 - styl_clsf_loss: 3.1829e-04 - val_loss: 4.1053 - val_dcd_sfmax_loss: 4.0908 - val_styl_clsf_loss: 1.4523e-04
Epoch 1/1
100/100 [==============================] - 1s 14ms/step - loss: 0.6945 - acc: 0.4903 - val_loss: 0.6947 - val_acc: 0.5625
Epoch 1/1
100/100 [==============================] - 12s 122ms/step - loss: 2.8534 - dcd_sfmax_loss: 2.8273 - styl_clsf_loss: 2.6097e-04 - val_loss: 4.0516 - val_dcd_sfmax_loss: 4.0447 - val_styl_clsf_loss: 6.8851e-05
Epoch 1/1
100/100 [==============================] - 34s 338ms/step - loss: 3.0329 - dcd_s[...]
{'fraction_non_alphanumeric': 0.16150234741784036, 'fraction_numerical': 0.07342723004694836, 'mean_word_length': 4.582809224318658, 'pattern_counts': {'":': 0, '<': 13, '<?xml version=': 0, '>': 22, 'https://': 3, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '3', 'dir': 'jupyter-structured-clean-dedup', 'id': '252987', 'license': 'MIT', 'n_tokens_mistral': 2230, 'n_tokens_neox': 1989, 'n_words': 382, 'path': 'notebooks/paraphraser_bible.ipynb', 'repo_name': 'delkind/paraphraser', 'size': '5325'}