<jupyter_start><jupyter_text>Setup<jupyter_code>import sys
import os

import re
import collections
import itertools
import bcolz
import pickle
sys.path.append('../../lib')

import numpy as np
import pandas as pd
import gc
import random
import smart_open
import h5py
import csv
import json
import functools
import time
import string

import datetime as dt
from tqdm import tqdm_notebook as tqdm

import numpy as np
import pandas as pd

import matplotlib.pyplot as plt
import seaborn as sns

import global_utils

random_state_number = 967898
import tensorflow as tf
from tensorflow.python.client import device_lib
def get_available_gpus():
    local_device_protos = device_lib.list_local_devices()
    return [x.name for x in local_device_protos if x.device_type == 'GPU']

config = tf.ConfigProto()
config.gpu_options.allow_growth=True
sess = tf.Session(config=config)
get_available_gpus()
%pylab
%matplotlib inline
%load_ext line_profiler
%load_ext memory_profiler
%load_ext autoreload
pd.options.mode.chained_assignment = None
pd.options.display.max_columns = 999
color = sns.color_palette()<jupyter_output><empty_output><jupyter_text>Data<jupyter_code>store = pd.HDFStore('../../data_prep/processed/stage1/data_frames.h5')
train_df = store['train_df']
test_df = store['test_df']
display(train_df.head())
display(test_df.head())
corpus_vocab_list, corpus_vocab_wordidx = None, None
with open('../../data_prep/processed/stage1/vocab_words_wordidx.pkl', 'rb') as f:
    (corpus_vocab_list, corpus_wordidx) = pickle.load(f)
print(len(corpus_vocab_list), len(corpus_wordidx))<jupyter_output>352220 352220<jupyter_text>Data Prep To control the vocabulary pass in updated corpus_wordidx<jupyter_code>from sklearn.model_selection import train_test_split
x_train_df, x_val_df = train_test_split(train_df,
                                         test_size=0.10, random_state=random_state_number,
                                         stratify=train_df.Class)

print(x_train_df.shape)
print(x_val_df.shape)
from tensorflow.contrib.keras.python.keras.utils import np_utils
from keras.preprocessing.sequence import pad_sequences
from keras.utils.np_utils import to_categorical
vocab_size=len(corpus_vocab_list)<jupyter_output><empty_output><jupyter_text>T:sent_words  generate data<jupyter_code>custom_unit_dict = {
         "gene_unit"      : "words",
         "variation_unit" : "words",
         # text transformed to sentences attribute
         "doc_unit"       : "words",
         "doc_form"       : "sentences",
         "divide_document": "multiple_unit"
      }
%autoreload
import global_utils
gen_data = global_utils.GenerateDataset(x_train_df, corpus_wordidx)
x_train_21_T, x_train_21_G, x_train_21_V, x_train_21_C = gen_data.generate_data(custom_unit_dict, 
                                                                             has_class=True,
                                                                             add_start_end_tag=True)
del gen_data
print("Train data")
print(np.array(x_train_21_T).shape, x_train_21_T[0])
print(np.array(x_train_21_G).shape, x_train_21_G[0])
print(np.array(x_train_21_V).shape, x_train_21_V[0])
print(np.array(x_train_21_C).shape, x_train_21_C[0])
gen_data = global_utils.GenerateDataset(x_val_df, corpus_wordidx)
x_val_21_T, x_val_21_G, x_val_21_V, x_val_21_C = gen_data.generate_data(custom_unit_dict, 
                                                                             has_class=True,
                                                                             add_start_end_tag=True)
del gen_data
print("Val data")
print("text",np.array(x_val_21_T).shape)
print("gene",np.array(x_val_21_G).shape, x_val_21_G[0])
print("variation",np.array(x_val_21_V).shape, x_val_21_V[0])
print("classes",np.array(x_val_21_C).shape, x_val_21_C[0])<jupyter_output>Val data
text (293702,)
gene (293702, 3) [364606, 112978, 364607]
variation (293702,) [364606, 295010, 364607]
classes (293702,) 2<jupyter_text>format data<jupyter_code>word_unknown_tag_idx   = corpus_wordidx["<UNK>"]
char_unknown_tag_idx   = global_utils.char_unknown_tag_idx
MAX_SENT_LEN = 60
x_train_21_T = pad_sequences(x_train_21_T, maxlen=MAX_SENT_LEN, value=word_unknown_tag_idx,
                                  padding="post",truncating="post")
x_val_21_T = pad_sequences(x_val_21_T, maxlen=MAX_SENT_LEN, value=word_unknown_tag_idx,
                                  padding="post",truncating="post")
print(x_train_21_T.shape, x_val_21_T.shape)<jupyter_output>(2622081, 60) (293702, 60)<jupyter_text>keras np_utils.to_categorical expects zero index categorical variableshttps://github.com/fchollet/keras/issues/570<jupyter_code>x_train_21_C = np.array(x_train_21_C) - 1
x_val_21_C = np.array(x_val_21_C) - 1
x_train_21_C = np_utils.to_categorical(np.array(x_train_21_C), 9)
x_val_21_C = np_utils.to_categorical(np.array(x_val_21_C), 9)
print(x_train_21_C.shape, x_val_21_C.shape)<jupyter_output>(2622081, 9) (293702, 9)<jupyter_text>T:text_words  generate data<jupyter_code>custom_unit_dict = {
         "gene_unit"      : "words",
         "variation_unit" : "words",
         # text transformed to sentences attribute
         "doc_unit"       : "words",
         "doc_form"       : "text",
         "divide_document": "single_unit"
      }
%autoreload
import global_utils
gen_data = global_utils.GenerateDataset(x_train_df, corpus_wordidx)
x_train_22_T, x_train_22_G, x_train_22_V, x_train_22_C = gen_data.generate_data(custom_unit_dict, 
                                                                             has_class=True,
                                                                             add_start_end_tag=True)
del gen_data
print("Train data")
print("text",np.array(x_train_22_T).shape)
print("gene",np.array(x_train_22_G).shape, x_train_22_G[0])
print("variation",np.array(x_train_22_V).shape, x_train_22_V[0])
print("classes",np.array(x_train_22_C).shape, x_train_22_C[0])
gen_data = global_utils.GenerateDataset(x_val_df, corpus_wordidx)
x_val_22_T, x_val_22_G, x_val_22_V, x_val_22_C = gen_data.generate_data(custom_unit_dict, 
                                                                             has_class=True,
                                                                             add_start_end_tag=True)
del gen_data
print("Val data")
print("text",np.array(x_val_22_T).shape)
print("gene",np.array(x_val_22_G).shape, x_val_22_G[0])
print("variation",np.array(x_val_22_V).shape, x_val_22_V[0])
print("classes",np.array(x_val_22_C).shape, x_val_22_C[0])<jupyter_output><empty_output><jupyter_text>format data<jupyter_code>word_unknown_tag_idx   = corpus_wordidx["<UNK>"]
char_unknown_tag_idx   = global_utils.char_unknown_tag_idx
MAX_TEXT_LEN = 5000
x_train_22_T = pad_sequences(x_train_22_T, maxlen=MAX_TEXT_LEN, value=word_unknown_tag_idx,
                                  padding="post",truncating="post")
x_val_22_T = pad_sequences(x_val_22_T, maxlen=MAX_TEXT_LEN, value=word_unknown_tag_idx,
                                  padding="post",truncating="post")
print(x_train_22_T.shape, x_val_22_T.shape)
MAX_GENE_LEN = 1
MAX_VAR_LEN = 4
x_train_22_G = pad_sequences(x_train_22_G, maxlen=MAX_GENE_LEN, value=word_unknown_tag_idx)
x_train_22_V = pad_sequences(x_train_22_V, maxlen=MAX_VAR_LEN, value=word_unknown_tag_idx)

x_val_22_G = pad_sequences(x_val_22_G, maxlen=MAX_GENE_LEN, value=word_unknown_tag_idx)
x_val_22_V = pad_sequences(x_val_22_V, maxlen=MAX_VAR_LEN, value=word_unknown_tag_idx)

print(x_train_22_G.shape, x_train_22_V.shape)
print(x_val_22_G.shape, x_val_22_V.shape)<jupyter_output>(2988, 1) (2988, 4)
(333, 1) (333, 4)<jupyter_text>keras np_utils.to_categorical expects zero index categorical variableshttps://github.com/fchollet/keras/issues/570<jupyter_code>x_train_22_C = np.array(x_train_22_C) - 1
x_val_22_C = np.array(x_val_22_C) - 1
x_train_22_C = np_utils.to_categorical(np.array(x_train_22_C), 9)
x_val_22_C = np_utils.to_categorical(np.array(x_val_22_C), 9)
print(x_train_22_C.shape, x_val_22_C.shape)<jupyter_output>(2988, 9) (333, 9)<jupyter_text>test Data setup<jupyter_code>gen_data = global_utils.GenerateDataset(test_df, corpus_wordidx)
x_test_22_T, x_test_22_G, x_test_22_V, _ = gen_data.generate_data(custom_unit_dict, 
                                                                has_class=False,
                                                                add_start_end_tag=True)
del gen_data
print("Test data")
print("text",np.array(x_test_22_T).shape)
print("gene",np.array(x_test_22_G).shape, x_test_22_G[0])
print("variation",np.array(x_test_22_V).shape, x_test_22_V[0])
x_test_22_T = pad_sequences(x_test_22_T, maxlen=MAX_TEXT_LEN, value=word_unknown_tag_idx,
                                  padding="post",truncating="post")
print(x_test_22_T.shape)
MAX_GENE_LEN = 1
MAX_VAR_LEN = 4
x_test_22_G = pad_sequences(x_test_22_G, maxlen=MAX_GENE_LEN, value=word_unknown_tag_idx)
x_test_22_V = pad_sequences(x_test_22_V, maxlen=MAX_VAR_LEN, value=word_unknown_tag_idx)

print(x_test_22_G.shape, x_test_22_V.shape)<jupyter_output>(986, 1) (986, 4)<jupyter_text>T:text_chars  generate data<jupyter_code>custom_unit_dict = {
         "gene_unit"          : "raw_chars",
         "variation_unit"     : "raw_chars",
         # text transformed to sentences attribute
         "doc_unit"           : "raw_chars",
         "doc_form"           : "text",
         "divide_document"    : "multiple_unit"
      }
%autoreload
import global_utils
gen_data = global_utils.GenerateDataset(x_train_df, corpus_wordidx)
x_train_33_T, x_train_33_G, x_train_33_V, x_train_33_C = gen_data.generate_data(custom_unit_dict, 
                                                                             has_class=True,
                                                                             add_start_end_tag=True)
del gen_data
print("Train data")
print("text",np.array(x_train_33_T).shape, x_train_33_T[0])
print("gene",np.array(x_train_33_G).shape, x_train_33_G[0])
print("variation",np.array(x_train_33_V).shape, x_train_33_V[0])
print("classes",np.array(x_train_33_C).shape, x_train_33_C[0])
%autoreload
import global_utils
gen_data = global_utils.GenerateDataset(x_val_df, corpus_wordidx)
x_val_33_T, x_val_33_G, x_val_33_V, x_val_33_C = gen_data.generate_data(custom_unit_dict, 
                                                                             has_class=True,
                                                                             add_start_end_tag=True)
del gen_data
print("Val data")
print("text",np.array(x_val_33_T).shape, x_val_33_T[98])
print("gene",np.array(x_val_33_G).shape, x_val_33_G[0])
print("variation",np.array(x_val_33_V).shape, x_val_33_V[0])
print("classes",np.array(x_val_33_C).shape, x_val_33_C[0])<jupyter_output>Val data
text (128341,) [74, 71, 0, 19, 72, 71, 19, 7, 8, 18, 72, 71, 19, 8, 12, 4, 72, 71, 15, 14, 8, 13, 19, 72, 71, 72, 71, 19, 7, 4, 72, 71, 4, 23, 15, 17, 4, 18, 18, 8, 14, 13, 72, 71, 14, 5, 72, 71, 22, 8, 11, 3, 36, 19, 24, 15, 4, 72, 71, 15, 27, 32, 8, 13, 10, 30, 0, 72, 71, 8, 13, 72, 71, 20, 28, 14, 18, 72, 71, 2, 4, 11, 11, 18, 72, 71, 8, 13, 3, 20, 2, 4, 3, 72, 71, 15, 14, 19, 4, 13, 19, 72, 71, 2, 4, 11, 11, 72, 71, 2, 24, 2, 11, 4, 72, 71, 0, 17, 17, 4, 18, 19, 72, 71, 0, 19, 72, 71, 1, 14, 19, 7, 72, 71, 19, 4, 12, 15, 4, 17, 0, 19, 20, 17, 4, 18, 72, 71, 72, 71, 15, 27, 32, 8, 13, 10, 30, 0, 72, 71, 8, 13, 3, 20, 2, 4, 3, 72, 71, 18, 36, 15, 7, 0, 18, 4, 72, 71, 8, 13, 7, 8, 1, 8, 19, 8, 14, 13, 72, 71, 14, 5, 72, 71, 30, 28, 33, 29, 72, 71, 72, 71, 0, 13, 3, 72, 71, 30, 35, 33, 29, 72, 71, 72, 71, 0, 19, 72, 71, 29, 33, 27, 2, 72, 71, 0, 13, 3, 72, 71, 30, 26, 27, 2, 72, 71, 72, 71, 17, 4, 18, 15, 4, 2, 19, 8, 21, 4, 11, 24, 72, 71, 72, 75]
gene (128341,) [74, 71, 2, 3[...]<jupyter_text>format data<jupyter_code>word_unknown_tag_idx   = corpus_wordidx["<UNK>"]
char_unknown_tag_idx   = global_utils.char_unknown_tag_idx
MAX_CHAR_IN_SENT_LEN = 150
x_train_33_T = pad_sequences(x_train_33_T, maxlen=MAX_CHAR_IN_SENT_LEN, value=char_unknown_tag_idx,
                                  padding="post",truncating="post")
x_val_33_T = pad_sequences(x_val_33_T, maxlen=MAX_CHAR_IN_SENT_LEN, value=char_unknown_tag_idx,
                                  padding="post",truncating="post")
print(x_train_33_T.shape, x_val_33_T.shape)
x_train_33_G = pad_sequences(x_train_33_G, maxlen=MAX_CHAR_IN_SENT_LEN, value=char_unknown_tag_idx)
x_train_33_V = pad_sequences(x_train_33_V, maxlen=MAX_CHAR_IN_SENT_LEN, value=char_unknown_tag_idx)

x_val_33_G = pad_sequences(x_val_33_G, maxlen=MAX_CHAR_IN_SENT_LEN, value=char_unknown_tag_idx)
x_val_33_V = pad_sequences(x_val_33_V, maxlen=MAX_CHAR_IN_SENT_LEN, value=char_unknown_tag_idx)

print(x_train_33_G.shape, x_train_33_V.shape)
print(x_val_33_G.shape, x_val_33_V.shape)<jupyter_output>(1086419, 150) (1086419, 150)
(128341, 150) (128341, 150)<jupyter_text>keras np_utils.to_categorical expects zero index categorical variableshttps://github.com/fchollet/keras/issues/570<jupyter_code>x_train_33_C = np.array(x_train_33_C) - 1
x_val_33_C = np.array(x_val_33_C) - 1
x_train_33_C = np_utils.to_categorical(np.array(x_train_33_C), 9)
x_val_33_C = np_utils.to_categorical(np.array(x_val_33_C), 9)
print(x_train_33_C.shape, x_val_33_C.shape)<jupyter_output>(1086419, 9) (128341, 9)<jupyter_text>T:text_sent_words  generate data<jupyter_code>custom_unit_dict = {
         "gene_unit"          : "words",
         "variation_unit"     : "words",
         # text transformed to sentences attribute
         "doc_unit"           : "word_list",
         "doc_form"           : "text",
         "divide_document"    : "single_unit"
      }
%autoreload
import global_utils
gen_data = global_utils.GenerateDataset(x_train_df, corpus_wordidx)
x_train_34_T, x_train_34_G, x_train_34_V, x_train_34_C = gen_data.generate_data(custom_unit_dict, 
                                                                             has_class=True,
                                                                             add_start_end_tag=True)
del gen_data
print("Train data")
print("text",np.array(x_train_34_T).shape, x_train_34_T[0][:1])
print("gene",np.array(x_train_34_G).shape, x_train_34_G[0])
print("variation",np.array(x_train_34_V).shape, x_train_34_V[0])
print("classes",np.array(x_train_34_C).shape, x_train_34_C[0])
%autoreload
import global_utils
gen_data = global_utils.GenerateDataset(x_val_df, corpus_wordidx)
x_val_34_T, x_val_34_G, x_val_34_V, x_val_34_C = gen_data.generate_data(custom_unit_dict, 
                                                                             has_class=True,
                                                                             add_start_end_tag=True)
del gen_data
print("Val data")
print("text",np.array(x_val_34_T).shape, x_val_34_T[98][:1])
print("gene",np.array(x_val_34_G).shape, x_val_34_G[0])
print("variation",np.array(x_val_34_V).shape, x_val_34_V[0])
print("classes",np.array(x_val_34_C).shape, x_val_34_C[0])<jupyter_output>Val data
text (333,) [[352216, 252037, 156537, 91785, 67201, 109857, 123191, 209585, 213751, 5638, 0, 126280, 49123, 331220, 0, 352217]]
gene (333, 3) [352216, 217983, 352217]
variation (333,) [352216, 41934, 352217]
classes (333,) 4<jupyter_text>format data<jupyter_code>word_unknown_tag_idx   = corpus_wordidx["<UNK>"]
char_unknown_tag_idx   = global_utils.char_unknown_tag_idx
MAX_DOC_LEN = 500 # no of sentences in a document
MAX_SENT_LEN = 80 # no of words in a sentence
for doc_i, doc in enumerate(x_train_34_T):
    x_train_34_T[doc_i] = x_train_34_T[doc_i][:MAX_DOC_LEN]
    # padding sentences
    if len(x_train_34_T[doc_i]) < MAX_DOC_LEN:
        for not_used_i in range(0,MAX_DOC_LEN - len(x_train_34_T[doc_i])):
            x_train_34_T[doc_i].append([word_unknown_tag_idx]*MAX_SENT_LEN)
    # padding words
    x_train_34_T[doc_i] = pad_sequences(x_train_34_T[doc_i], maxlen=MAX_SENT_LEN, value=word_unknown_tag_idx)
    
for doc_i, doc in enumerate(x_val_34_T):
    x_val_34_T[doc_i] = x_val_34_T[doc_i][:MAX_DOC_LEN]
    # padding sentences
    if len(x_val_34_T[doc_i]) < MAX_DOC_LEN:
        for not_used_i in range(0,MAX_DOC_LEN - len(x_val_34_T[doc_i])):
            x_val_34_T[doc_i].append([word_unknown_tag_idx]*MAX_SENT_LEN)
    # padding words
    x_val_34_T[doc_i] = pad_sequences(x_val_34_T[doc_i], maxlen=MAX_SENT_LEN, value=word_unknown_tag_idx)
    
x_train_34_T = np.array(x_train_34_T)
x_val_34_T = np.array(x_val_34_T)
print(x_val_34_T.shape, x_train_34_T.shape)
x_train_34_G = pad_sequences(x_train_34_G, maxlen=MAX_SENT_LEN, value=word_unknown_tag_idx)
x_train_34_V = pad_sequences(x_train_34_V, maxlen=MAX_SENT_LEN, value=word_unknown_tag_idx)

x_val_34_G = pad_sequences(x_val_34_G, maxlen=MAX_SENT_LEN, value=word_unknown_tag_idx)
x_val_34_V = pad_sequences(x_val_34_V, maxlen=MAX_SENT_LEN, value=word_unknown_tag_idx)

print(x_train_34_G.shape, x_train_34_V.shape)
print(x_val_34_G.shape, x_val_34_V.shape)<jupyter_output>(2988, 80) (2988, 80)
(333, 80) (333, 80)<jupyter_text>keras np_utils.to_categorical expects zero index categorical variableshttps://github.com/fchollet/keras/issues/570<jupyter_code>x_train_34_C = np.array(x_train_34_C) - 1
x_val_34_C = np.array(x_val_34_C) - 1
x_train_34_C = np_utils.to_categorical(np.array(x_train_34_C), 9)
x_val_34_C = np_utils.to_categorical(np.array(x_val_34_C), 9)
print(x_train_34_C.shape, x_val_34_C.shape)<jupyter_output>(2988, 9) (333, 9)<jupyter_text>Need to form 3 dimensional target data for rationale model training<jupyter_code>temp = (x_train_34_C.shape[0],1,x_train_34_C.shape[1])
x_train_34_C_sent = np.repeat(x_train_34_C.reshape(temp[0],temp[1],temp[2]), MAX_DOC_LEN, axis=1)

#sentence test targets
temp = (x_val_34_C.shape[0],1,x_val_34_C.shape[1])
x_val_34_C_sent = np.repeat(x_val_34_C.reshape(temp[0],temp[1],temp[2]), MAX_DOC_LEN, axis=1)

print(x_train_34_C_sent.shape, x_val_34_C_sent.shape)<jupyter_output>(2988, 500, 9) (333, 500, 9)<jupyter_text>Embedding layer  for words<jupyter_code>WORD_EMB_SIZE = 200
%autoreload
import global_utils
ft_file_path = "/home/bicepjai/Projects/Deep-Survey-Text-Classification/data_prep/processed/stage1/pretrained_word_vectors/ft_sg_200d_50e.vec"
trained_embeddings = global_utils.get_embeddings_from_ft(ft_file_path, WORD_EMB_SIZE, corpus_vocab_list)
trained_embeddings.shape<jupyter_output><empty_output><jupyter_text>for characters<jupyter_code>CHAR_EMB_SIZE = 64
char_embeddings = np.random.randn(global_utils.CHAR_ALPHABETS_LEN, CHAR_EMB_SIZE)
char_embeddings.shape<jupyter_output><empty_output><jupyter_text>Models  prep<jupyter_code>%autoreload
import tensorflow.contrib.keras as keras
import tensorflow as tf

from keras import backend as K

from keras.engine import Layer, InputSpec, InputLayer

from keras.models import Model, Sequential

from keras.layers import Dropout, Embedding, concatenate
from keras.layers import Conv1D, MaxPooling1D, Conv2D, MaxPooling2D, ZeroPadding1D
from keras.layers import Dense, Input, Flatten, BatchNormalization
from keras.layers import Concatenate, Dot, Merge, Multiply, RepeatVector
from keras.layers import Bidirectional, TimeDistributed
from keras.layers import SimpleRNN, LSTM, GRU, Lambda, Permute

from keras.layers.core import Reshape, Activation
from keras.optimizers import Adam
from keras.callbacks import ModelCheckpoint,EarlyStopping,TensorBoard
from keras.constraints import maxnorm
from keras.regularizers import l2<jupyter_output><empty_output><jupyter_text>model_1: paper refer https://github.com/bwallace/rationale-CNN  Doc-CNN  model<jupyter_code>doc_input = Input(shape=(MAX_DOC_LEN,MAX_SENT_LEN,), dtype="int16")
reshape_1d = Reshape([MAX_DOC_LEN * MAX_SENT_LEN])(doc_input)
doc_embedding_1d = Embedding(vocab_size, WORD_EMB_SIZE, weights=[trained_embeddings], trainable=True)(reshape_1d)
# data_format='channels_first' for conv2d
doc_embedding = Reshape([1, MAX_DOC_LEN, MAX_SENT_LEN * WORD_EMB_SIZE])(doc_embedding_1d)

sent_convs_in_doc = []
ngram_filters = [2,3]
n_filters = 32

# using Conv2D instead of Conv1D since we need to deal with sentences and not the whole document
# All Input shape: 4D tensor with shape: (samples, channels, rows, cols) if data_format='channels_first'
for n_gram in ngram_filters:
    l_conv = Conv2D(filters = n_filters, 
                    kernel_size = (1, n_gram * WORD_EMB_SIZE), # n_gram words
                    strides = (1, WORD_EMB_SIZE), # one word
                    data_format='channels_first',
                    activation="relu")(doc_embedding)
    # this output (n_filters x max_doc_len x 1)
    l_pool = MaxPooling2D(pool_size=(1, (MAX_SENT_LEN - n_gram + 1)),
                       data_format='channels_first')(l_conv)
    
    # flip around, to get (1 x DOC_SEQ_LEN x n_filters)
    permuted = Permute((2,1,3)) (l_pool)
    
    # drop extra dimension
    reshaped = Reshape((MAX_DOC_LEN, n_filters))(permuted)
    sent_convs_in_doc.append(reshaped)
    
l_concat = Concatenate(axis=1)(sent_convs_in_doc)
l_dropout = Dropout(0.5)(l_concat)
                       
def sum_sent_vecs(x):
    return K.sum(x, axis=1)

def sum_sent_vec_output_shape(input_shape): 
    # should be (batch x MAX_DOC_LEN x MAX_SENT_LEN)
    shape = list(input_shape) 
    # something like (None, 96), where 96 is the
    # length of induced sentence vectors
    return (shape[0], shape[-1])

doc_vector = Lambda(sum_sent_vecs, 
                    output_shape=sum_sent_vec_output_shape)(l_dropout)
l_softmax = Dense(9, activation='softmax')(doc_vector)
doc_cnn_model = Model(inputs=[doc_input], outputs=[l_softmax])<jupyter_output><empty_output><jupyter_text>training<jupyter_code>doc_cnn_model.compile(loss='categorical_crossentropy', optimizer=Adam(), metrics=['categorical_accuracy'])
doc_cnn_model.summary()
%rm -rf ./tb_graphs/*
tb_callback = keras.callbacks.TensorBoard(log_dir='./tb_graphs', histogram_freq=0, write_graph=True, write_images=True)
checkpointer = ModelCheckpoint(filepath="doc_cnn_weights.hdf5", 
                                    verbose=1,
                                    monitor="val_categorical_accuracy",
                                    save_best_only=True,
                                    mode="max")
with tf.Session() as sess:
    # model = keras.models.load_model('current_model.h5')
    sess.run(tf.global_variables_initializer())
    try:
        doc_cnn_model.load_weights("model_weights.hdf5")
    except IOError as ioe:
        print("no checkpoints available !")
    doc_cnn_model.fit(x_train_34_T, x_train_34_C, 
          validation_data=(x_val_34_T, x_val_34_C),
          epochs=5, batch_size=32, shuffle=True,
          callbacks=[tb_callback,checkpointer])
    #model.save('current_sent_model.h5')<jupyter_output>no checkpoints available !
Train on 2988 samples, validate on 333 samples
Epoch 1/5
2976/2988 [============================>.] - ETA: 0s - loss: 9.1418 - categorical_accuracy: 0.2853Epoch 00000: val_categorical_accuracy improved from -inf to 0.37838, saving model to doc_cnn_weights.hdf5
2988/2988 [==============================] - 51s - loss: 9.1449 - categorical_accuracy: 0.2855 - val_loss: 8.5094 - val_categorical_accuracy: 0.3784
Epoch 2/5
2976/2988 [============================>.] - ETA: 0s - loss: 5.4533 - categorical_accuracy: 0.4385Epoch 00001: val_categorical_accuracy improved from 0.37838 to 0.56156, saving model to doc_cnn_weights.hdf5
2988/2988 [==============================] - 44s - loss: 5.4415 - categorical_accuracy: 0.4391 - val_loss: 1.6307 - val_categorical_accuracy: 0.5616
Epoch 3/5
2976/2988 [============================>.] - ETA: 0s - loss: 1.1139 - categorical_accuracy: 0.6546Epoch 00002: val_categorical_accuracy improved from 0.56156 to 0.61862, saving model to d[...]<jupyter_text>RA-CNN  model<jupyter_code>doc_input = Input(shape=(MAX_DOC_LEN,MAX_SENT_LEN,), dtype="int32")
reshape_1d = Reshape([MAX_DOC_LEN * MAX_SENT_LEN])(doc_input)
doc_embedding_1d = Embedding(vocab_size, WORD_EMB_SIZE, weights=[trained_embeddings], trainable=True)(reshape_1d)
# data_format='channels_first' for conv2d
doc_embedding = Reshape([1, MAX_DOC_LEN, MAX_SENT_LEN * WORD_EMB_SIZE])(doc_embedding_1d)

sent_convs_in_doc = []
ngram_filters = [2,3]
n_filters = 32 # nof features
final_doc_dims = len(ngram_filters) * n_filters 

# using Conv2D instead of Conv1D since we need to deal with sentences and not the whole document
# All Input shape: 4D tensor with shape: (samples, channels, rows, cols) if data_format='channels_first'
for n_gram in ngram_filters:
    l_conv = Conv2D(filters = n_filters, 
                    kernel_size = (1, n_gram * WORD_EMB_SIZE), # n_gram words
                    strides = (1, WORD_EMB_SIZE), # one word
                    data_format='channels_first',
                    activation="relu")(doc_embedding)
    # this output (n_filters x max_doc_len x 1)
    l_pool = MaxPooling2D(pool_size=(1, (MAX_SENT_LEN - n_gram + 1)),
                       data_format='channels_first')(l_conv)
    
    # flip around, to get (1 x DOC_SEQ_LEN x n_filters)
    permuted = Permute((2,1,3)) (l_pool)
    
    # drop extra dimension
    reshaped = Reshape((MAX_DOC_LEN, n_filters))(permuted)
    sent_convs_in_doc.append(reshaped)
    
sent_vectors = concatenate(sent_convs_in_doc)
# do we need dropout here, we might lose information
# l_dropout = Dropout(0.5)(l_concat)

sentence_softmax = Dense(9, activation='softmax', kernel_regularizer=l2(0.01), name="sentence_prediction")
doc_sent_output_layer = TimeDistributed(sentence_softmax, name="sentence_predictions")(sent_vectors)

# weights are set to the estimated probabilities that 
# corresponding sentences are rationales in the most likely direction
sum_weighting_probs = Lambda(lambda x: K.max(x, axis=1))

# distributing over sentences in the document
sent_weights = TimeDistributed(sum_weighting_probs)(doc_sent_output_layer)

# reshaping the weights to perform matrix dot product
reshaped_sent_weights = Reshape((1, MAX_DOC_LEN))(sent_weights)

# along the last 2 axes and not including the batch axis
doc_vector = Dot((1,2))([sent_vectors, reshaped_sent_weights])
doc_vector = Reshape((final_doc_dims,))(doc_vector)

l_dropout = Dropout(0.5)(doc_vector)
doc_output_layer = Dense(9, activation="softmax")(l_dropout)
        
sentence_model = Model(inputs=doc_input, outputs=doc_sent_output_layer)
sentence_model.summary()
ra_cnn_model = Model(inputs=doc_input, outputs=doc_output_layer)
ra_cnn_model.summary()<jupyter_output>____________________________________________________________________________________________________
Layer (type)                     Output Shape          Param #     Connected to                     
====================================================================================================
input_1 (InputLayer)             (None, 500, 80)       0                                            
____________________________________________________________________________________________________
reshape_1 (Reshape)              (None, 40000)         0           input_1[0][0]                    
____________________________________________________________________________________________________
embedding_1 (Embedding)          (None, 40000, 200)    70444000    reshape_1[0][0]                  
____________________________________________________________________________________________________
reshape_2 (Reshape)              (None, 1, 500, 16000) 0           embedding_1[0][0]       [...]<jupyter_text>training sentence model<jupyter_code>sentence_model.compile(loss='categorical_crossentropy', optimizer=Adam(), metrics=['categorical_accuracy'])<jupyter_output><empty_output><jupyter_text>the accuracy is not improving anymore<jupyter_code>%rm -rf ./tb_graphs/*
tb_callback = keras.callbacks.TensorBoard(log_dir='./tb_graphs', histogram_freq=0, write_graph=True, write_images=True)
checkpointer = ModelCheckpoint(filepath="sentence_model_weights.hdf5", 
                                    verbose=1,
                                    monitor="val_categorical_accuracy",
                                    save_best_only=True,
                                    mode="max")
with tf.Session() as sess:
    # model = keras.models.load_model('current_model.h5')
    sess.run(tf.global_variables_initializer())
    try:
        sentence_model.load_weights("sentence_model_weights.hdf5")
    except IOError as ioe:
        print("no checkpoints available !")
    sentence_model.fit(x_train_34_T, x_train_34_C_sent, 
          validation_data=(x_val_34_T, x_val_34_C_sent),
          epochs=10, batch_size=32, shuffle=True,
          callbacks=[tb_callback,checkpointer])
    #sentence_model.save('current_sent_model.h5')
with tf.Session() as sess:
    # model = keras.models.load_model('current_model.h5')
    sess.run(tf.global_variables_initializer())
    try:
        sentence_model.load_weights("sentence_model_weights.hdf5")
    except IOError as ioe:
        print("no checkpoints available !")
    sentence_model.fit(x_train_34_T, x_train_34_C_sent, 
          validation_data=(x_val_34_T, x_val_34_C_sent),
          epochs=10, batch_size=32, shuffle=True,
          callbacks=[tb_callback,checkpointer])
    #sentence_model.save('current_sent_model.h5')<jupyter_output>Train on 2988 samples, validate on 333 samples
Epoch 1/10
2976/2988 [============================>.] - ETA: 0s - loss: 1.3352 - categorical_accuracy: 0.4924Epoch 00000: val_categorical_accuracy did not improve
2988/2988 [==============================] - 41s - loss: 1.3344 - categorical_accuracy: 0.4926 - val_loss: 1.6008 - val_categorical_accuracy: 0.4330
Epoch 2/10
2976/2988 [============================>.] - ETA: 0s - loss: 1.3199 - categorical_accuracy: 0.4996Epoch 00001: val_categorical_accuracy did not improve
2988/2988 [==============================] - 41s - loss: 1.3209 - categorical_accuracy: 0.4993 - val_loss: 1.6259 - val_categorical_accuracy: 0.4326
Epoch 3/10
2976/2988 [============================>.] - ETA: 0s - loss: 1.3099 - categorical_accuracy: 0.5022Epoch 00002: val_categorical_accuracy did not improve
2988/2988 [==============================] - 41s - loss: 1.3105 - categorical_accuracy: 0.5021 - val_loss: 1.6049 - val_categorical_accuracy: 0.4299
Epoch 4/10
2976/2[...]<jupyter_text>training ra_cnn model we are freezing the sentence level trained model which will be used as input for rationale model<jupyter_code>sentence_model.load_weights("sentence_model_weights.hdf5")
sent_softmax_layer = ra_cnn_model.get_layer("sentence_predictions")
sent_softmax_layer.trainable = False
ra_cnn_model.compile(loss='categorical_crossentropy', optimizer=Adam(), metrics=['categorical_accuracy'])
%rm -rf ./tb_graphs/*
tb_callback = keras.callbacks.TensorBoard(log_dir='./tb_graphs', histogram_freq=0, write_graph=True, write_images=True)
checkpointer = ModelCheckpoint(filepath="ra_cnn_model_weights.hdf5", 
                                    verbose=1,
                                    monitor="val_categorical_accuracy",
                                    save_best_only=True,
                                    mode="max")
with tf.Session() as sess:
    # ra_cnn_model = keras.models.load_model('current_model.h5')
    sess.run(tf.global_variables_initializer())
    try:
        ra_cnn_model.load_weights("ra_cnn_model_weights.hdf5")
    except IOError as ioe:
        print("no checkpoints available !")
    ra_cnn_model.fit(x_train_34_T, x_train_34_C, 
          validation_data=(x_val_34_T, x_val_34_C),
          epochs=10, batch_size=32, shuffle=True,
          callbacks=[tb_callback,checkpointer])
    #ra_cnn_model.save('current_sent_model.h5')<jupyter_output>no checkpoints available !
Train on 2988 samples, validate on 333 samples
Epoch 1/10
2976/2988 [============================>.] - ETA: 0s - loss: 1.8309 - categorical_accuracy: 0.4052Epoch 00000: val_categorical_accuracy improved from -inf to 0.57057, saving model to ra_cnn_model_weights.hdf5
2988/2988 [==============================] - 50s - loss: 1.8302 - categorical_accuracy: 0.4053 - val_loss: 1.3053 - val_categorical_accuracy: 0.5706
Epoch 2/10
2976/2988 [============================>.] - ETA: 0s - loss: 1.2696 - categorical_accuracy: 0.5985Epoch 00001: val_categorical_accuracy improved from 0.57057 to 0.62162, saving model to ra_cnn_model_weights.hdf5
2988/2988 [==============================] - 43s - loss: 1.2717 - categorical_accuracy: 0.5971 - val_loss: 1.1158 - val_categorical_accuracy: 0.6216
Epoch 3/10
2976/2988 [============================>.] - ETA: 0s - loss: 0.9867 - categorical_accuracy: 0.6838Epoch 00002: val_categorical_accuracy improved from 0.62162 to 0.65766, savi[...]
{'fraction_non_alphanumeric': 0.121918353596913, 'fraction_numerical': 0.06651762472054636, 'mean_word_length': 4.017516902274124, 'pattern_counts': {'":': 2, '<': 93, '<?xml version=': 0, '>': 100, 'https://': 5, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 106, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '26', 'dir': 'jupyter-structured-clean-dedup', 'id': '83837', 'license': 'MIT', 'n_tokens_mistral': 13546, 'n_tokens_neox': 11842, 'n_words': 2056, 'path': 'deep_models/paper_05_racnn/models.ipynb', 'repo_name': 'bicepjai/Deep-Survey-on-Text-Classification', 'size': '32653'}