<jupyter_start><jupyter_text>Setup<jupyter_code>import sys
import os

import re
import collections
import itertools
import bcolz
import pickle
sys.path.append('../../lib')
sys.path.append('../')

import numpy as np
import pandas as pd
import gc
import random
import smart_open
import h5py
import csv
import json
import functools
import time
import string

import datetime as dt
from tqdm import tqdm_notebook as tqdm

import numpy as np
import pandas as pd

import matplotlib.pyplot as plt
import seaborn as sns

import global_utils

random_state_number = 967898
import tensorflow as tf
from tensorflow.python.client import device_lib
def get_available_gpus():
    local_device_protos = device_lib.list_local_devices()
    return [x.name for x in local_device_protos if x.device_type == 'GPU']

config = tf.ConfigProto()
config.gpu_options.allow_growth=True
sess = tf.Session(config=config)
get_available_gpus()
%pylab
%matplotlib inline
%load_ext line_profiler
%load_ext memory_profiler
%load_ext autoreload
pd.options.mode.chained_assignment = None
pd.options.display.max_columns = 999
color = sns.color_palette()<jupyter_output><empty_output><jupyter_text>Data<jupyter_code>store = pd.HDFStore('../../data_prep/processed/stage1/data_frames.h5')
train_df = store['train_df']
test_df = store['test_df']
display(train_df.head())
display(test_df.head())
corpus_vocab_list, corpus_vocab_wordidx = None, None
with open('../../data_prep/processed/stage1/vocab_words_wordidx.pkl', 'rb') as f:
    (corpus_vocab_list, corpus_wordidx) = pickle.load(f)
print(len(corpus_vocab_list), len(corpus_wordidx))<jupyter_output>352220 352220<jupyter_text>Data Prep To control the vocabulary pass in updated corpus_wordidx<jupyter_code>from sklearn.model_selection import train_test_split
x_train_df, x_val_df = train_test_split(train_df,
                                         test_size=0.10, random_state=random_state_number,
                                         stratify=train_df.Class)

print(x_train_df.shape)
print(x_val_df.shape)
from tensorflow.contrib.keras.python.keras.utils import np_utils
from keras.preprocessing.sequence import pad_sequences
from keras.utils.np_utils import to_categorical
vocab_size=len(corpus_vocab_list)<jupyter_output><empty_output><jupyter_text>T:sent_words  generate data<jupyter_code>custom_unit_dict = {
         "gene_unit"      : "words",
         "variation_unit" : "words",
         # text transformed to sentences attribute
         "doc_unit"       : "words",
         "doc_form"       : "sentences",
         "divide_document": "multiple_unit"
      }
%autoreload
import global_utils
gen_data = global_utils.GenerateDataset(x_train_df, corpus_wordidx)
x_train_21_T, x_train_21_G, x_train_21_V, x_train_21_C = gen_data.generate_data(custom_unit_dict, 
                                                                             has_class=True,
                                                                             add_start_end_tag=True)
del gen_data
print("Train data")
print(np.array(x_train_21_T).shape, x_train_21_T[0])
print(np.array(x_train_21_G).shape, x_train_21_G[0])
print(np.array(x_train_21_V).shape, x_train_21_V[0])
print(np.array(x_train_21_C).shape, x_train_21_C[0])
gen_data = global_utils.GenerateDataset(x_val_df, corpus_wordidx)
x_val_21_T, x_val_21_G, x_val_21_V, x_val_21_C = gen_data.generate_data(custom_unit_dict, 
                                                                             has_class=True,
                                                                             add_start_end_tag=True)
del gen_data
print("Val data")
print("text",np.array(x_val_21_T).shape)
print("gene",np.array(x_val_21_G).shape, x_val_21_G[0])
print("variation",np.array(x_val_21_V).shape, x_val_21_V[0])
print("classes",np.array(x_val_21_C).shape, x_val_21_C[0])<jupyter_output>Val data
text (128341,)
gene (128341, 3) [352216, 217983, 352217]
variation (128341,) [352216, 41934, 352217]
classes (128341,) 4<jupyter_text>format data<jupyter_code>word_unknown_tag_idx   = corpus_wordidx["<UNK>"]
char_unknown_tag_idx   = global_utils.char_unknown_tag_idx
MAX_SENT_LEN = 60
x_train_21_T = pad_sequences(x_train_21_T, maxlen=MAX_SENT_LEN, value=word_unknown_tag_idx,
                                  padding="post",truncating="post")
x_val_21_T = pad_sequences(x_val_21_T, maxlen=MAX_SENT_LEN, value=word_unknown_tag_idx,
                                  padding="post",truncating="post")
print(x_train_21_T.shape, x_val_21_T.shape)<jupyter_output>(1086419, 60) (128341, 60)<jupyter_text>keras np_utils.to_categorical expects zero index categorical variableshttps://github.com/fchollet/keras/issues/570<jupyter_code>x_train_21_C = np.array(x_train_21_C) - 1
x_val_21_C = np.array(x_val_21_C) - 1
x_train_21_C = np_utils.to_categorical(np.array(x_train_21_C), 9)
x_val_21_C = np_utils.to_categorical(np.array(x_val_21_C), 9)
print(x_train_21_C.shape, x_val_21_C.shape)<jupyter_output>(1086419, 9) (128341, 9)<jupyter_text>T:text_words  generate data<jupyter_code>custom_unit_dict = {
         "gene_unit"      : "words",
         "variation_unit" : "words",
         # text transformed to sentences attribute
         "doc_unit"       : "words",
         "doc_form"       : "text",
         "divide_document": "single_unit"
      }
%autoreload
import global_utils
gen_data = global_utils.GenerateDataset(x_train_df, corpus_wordidx)
x_train_22_T, x_train_22_G, x_train_22_V, x_train_22_C = gen_data.generate_data(custom_unit_dict, 
                                                                             has_class=True,
                                                                             add_start_end_tag=True)
del gen_data
print("Train data")
print("text",np.array(x_train_22_T).shape)
print("gene",np.array(x_train_22_G).shape, x_train_22_G[0])
print("variation",np.array(x_train_22_V).shape, x_train_22_V[0])
print("classes",np.array(x_train_22_C).shape, x_train_22_C[0])
gen_data = global_utils.GenerateDataset(x_val_df, corpus_wordidx)
x_val_22_T, x_val_22_G, x_val_22_V, x_val_22_C = gen_data.generate_data(custom_unit_dict, 
                                                                             has_class=True,
                                                                             add_start_end_tag=True)
del gen_data
print("Val data")
print("text",np.array(x_val_22_T).shape)
print("gene",np.array(x_val_22_G).shape, x_val_22_G[0])
print("variation",np.array(x_val_22_V).shape, x_val_22_V[0])
print("classes",np.array(x_val_22_C).shape, x_val_22_C[0])<jupyter_output>Val data
text (333,)
gene (333, 3) [352216, 217983, 352217]
variation (333,) [352216, 41934, 352217]
classes (333,) 4<jupyter_text>format data<jupyter_code>word_unknown_tag_idx   = corpus_wordidx["<UNK>"]
char_unknown_tag_idx   = global_utils.char_unknown_tag_idx
MAX_TEXT_LEN = 5000
x_train_22_T = pad_sequences(x_train_22_T, maxlen=MAX_TEXT_LEN, value=word_unknown_tag_idx,
                                  padding="post",truncating="post")
x_val_22_T = pad_sequences(x_val_22_T, maxlen=MAX_TEXT_LEN, value=word_unknown_tag_idx,
                                  padding="post",truncating="post")
print(x_train_22_T.shape, x_val_22_T.shape)
MAX_GENE_LEN = 1
MAX_VAR_LEN = 4
x_train_22_G = pad_sequences(x_train_22_G, maxlen=MAX_GENE_LEN, value=word_unknown_tag_idx)
x_train_22_V = pad_sequences(x_train_22_V, maxlen=MAX_VAR_LEN, value=word_unknown_tag_idx)

x_val_22_G = pad_sequences(x_val_22_G, maxlen=MAX_GENE_LEN, value=word_unknown_tag_idx)
x_val_22_V = pad_sequences(x_val_22_V, maxlen=MAX_VAR_LEN, value=word_unknown_tag_idx)

print(x_train_22_G.shape, x_train_22_V.shape)
print(x_val_22_G.shape, x_val_22_V.shape)<jupyter_output>(2988, 1) (2988, 4)
(333, 1) (333, 4)<jupyter_text>keras np_utils.to_categorical expects zero index categorical variableshttps://github.com/fchollet/keras/issues/570<jupyter_code>x_train_22_C = np.array(x_train_22_C) - 1
x_val_22_C = np.array(x_val_22_C) - 1
x_train_22_C = np_utils.to_categorical(np.array(x_train_22_C), 9)
x_val_22_C = np_utils.to_categorical(np.array(x_val_22_C), 9)
print(x_train_22_C.shape, x_val_22_C.shape)<jupyter_output>(2988, 9) (333, 9)<jupyter_text>test Data setup<jupyter_code>gen_data = global_utils.GenerateDataset(test_df, corpus_wordidx)
x_test_22_T, x_test_22_G, x_test_22_V, _ = gen_data.generate_data(custom_unit_dict, 
                                                                has_class=False,
                                                                add_start_end_tag=True)
del gen_data
print("Test data")
print("text",np.array(x_test_22_T).shape)
print("gene",np.array(x_test_22_G).shape, x_test_22_G[0])
print("variation",np.array(x_test_22_V).shape, x_test_22_V[0])
x_test_22_T = pad_sequences(x_test_22_T, maxlen=MAX_TEXT_LEN, value=word_unknown_tag_idx,
                                  padding="post",truncating="post")
print(x_test_22_T.shape)
MAX_GENE_LEN = 1
MAX_VAR_LEN = 4
x_test_22_G = pad_sequences(x_test_22_G, maxlen=MAX_GENE_LEN, value=word_unknown_tag_idx)
x_test_22_V = pad_sequences(x_test_22_V, maxlen=MAX_VAR_LEN, value=word_unknown_tag_idx)

print(x_test_22_G.shape, x_test_22_V.shape)<jupyter_output>(986, 1) (986, 4)<jupyter_text>T:text_chars  generate data<jupyter_code>custom_unit_dict = {
         "gene_unit"          : "raw_chars",
         "variation_unit"     : "raw_chars",
         # text transformed to sentences attribute
         "doc_unit"           : "raw_chars",
         "doc_form"           : "text",
         "divide_document"    : "multiple_unit"
      }
%autoreload
import global_utils
gen_data = global_utils.GenerateDataset(x_train_df, corpus_wordidx)
x_train_33_T, x_train_33_G, x_train_33_V, x_train_33_C = gen_data.generate_data(custom_unit_dict, 
                                                                             has_class=True,
                                                                             add_start_end_tag=True)
del gen_data
print("Train data")
print("text",np.array(x_train_33_T).shape, x_train_33_T[0])
print("gene",np.array(x_train_33_G).shape, x_train_33_G[0])
print("variation",np.array(x_train_33_V).shape, x_train_33_V[0])
print("classes",np.array(x_train_33_C).shape, x_train_33_C[0])
%autoreload
import global_utils
gen_data = global_utils.GenerateDataset(x_val_df, corpus_wordidx)
x_val_33_T, x_val_33_G, x_val_33_V, x_val_33_C = gen_data.generate_data(custom_unit_dict, 
                                                                             has_class=True,
                                                                             add_start_end_tag=True)
del gen_data
print("Val data")
print("text",np.array(x_val_33_T).shape, x_val_33_T[98])
print("gene",np.array(x_val_33_G).shape, x_val_33_G[0])
print("variation",np.array(x_val_33_V).shape, x_val_33_V[0])
print("classes",np.array(x_val_33_C).shape, x_val_33_C[0])<jupyter_output>Val data
text (128341,) [74, 71, 0, 19, 72, 71, 19, 7, 8, 18, 72, 71, 19, 8, 12, 4, 72, 71, 15, 14, 8, 13, 19, 72, 71, 72, 71, 19, 7, 4, 72, 71, 4, 23, 15, 17, 4, 18, 18, 8, 14, 13, 72, 71, 14, 5, 72, 71, 22, 8, 11, 3, 36, 19, 24, 15, 4, 72, 71, 15, 27, 32, 8, 13, 10, 30, 0, 72, 71, 8, 13, 72, 71, 20, 28, 14, 18, 72, 71, 2, 4, 11, 11, 18, 72, 71, 8, 13, 3, 20, 2, 4, 3, 72, 71, 15, 14, 19, 4, 13, 19, 72, 71, 2, 4, 11, 11, 72, 71, 2, 24, 2, 11, 4, 72, 71, 0, 17, 17, 4, 18, 19, 72, 71, 0, 19, 72, 71, 1, 14, 19, 7, 72, 71, 19, 4, 12, 15, 4, 17, 0, 19, 20, 17, 4, 18, 72, 71, 72, 71, 15, 27, 32, 8, 13, 10, 30, 0, 72, 71, 8, 13, 3, 20, 2, 4, 3, 72, 71, 18, 36, 15, 7, 0, 18, 4, 72, 71, 8, 13, 7, 8, 1, 8, 19, 8, 14, 13, 72, 71, 14, 5, 72, 71, 30, 28, 33, 29, 72, 71, 72, 71, 0, 13, 3, 72, 71, 30, 35, 33, 29, 72, 71, 72, 71, 0, 19, 72, 71, 29, 33, 27, 2, 72, 71, 0, 13, 3, 72, 71, 30, 26, 27, 2, 72, 71, 72, 71, 17, 4, 18, 15, 4, 2, 19, 8, 21, 4, 11, 24, 72, 71, 72, 75]
gene (128341,) [74, 71, 2, 3[...]<jupyter_text>format data<jupyter_code>word_unknown_tag_idx   = corpus_wordidx["<UNK>"]
char_unknown_tag_idx   = global_utils.char_unknown_tag_idx
MAX_CHAR_IN_SENT_LEN = 150
x_train_33_T = pad_sequences(x_train_33_T, maxlen=MAX_CHAR_IN_SENT_LEN, value=char_unknown_tag_idx,
                                  padding="post",truncating="post")
x_val_33_T = pad_sequences(x_val_33_T, maxlen=MAX_CHAR_IN_SENT_LEN, value=char_unknown_tag_idx,
                                  padding="post",truncating="post")
print(x_train_33_T.shape, x_val_33_T.shape)
x_train_33_G = pad_sequences(x_train_33_G, maxlen=MAX_CHAR_IN_SENT_LEN, value=char_unknown_tag_idx)
x_train_33_V = pad_sequences(x_train_33_V, maxlen=MAX_CHAR_IN_SENT_LEN, value=char_unknown_tag_idx)

x_val_33_G = pad_sequences(x_val_33_G, maxlen=MAX_CHAR_IN_SENT_LEN, value=char_unknown_tag_idx)
x_val_33_V = pad_sequences(x_val_33_V, maxlen=MAX_CHAR_IN_SENT_LEN, value=char_unknown_tag_idx)

print(x_train_33_G.shape, x_train_33_V.shape)
print(x_val_33_G.shape, x_val_33_V.shape)<jupyter_output>(1086419, 150) (1086419, 150)
(128341, 150) (128341, 150)<jupyter_text>keras np_utils.to_categorical expects zero index categorical variableshttps://github.com/fchollet/keras/issues/570<jupyter_code>x_train_33_C = np.array(x_train_33_C) - 1
x_val_33_C = np.array(x_val_33_C) - 1
x_train_33_C = np_utils.to_categorical(np.array(x_train_33_C), 9)
x_val_33_C = np_utils.to_categorical(np.array(x_val_33_C), 9)
print(x_train_33_C.shape, x_val_33_C.shape)<jupyter_output>(1086419, 9) (128341, 9)<jupyter_text>T:text_sent_words  generate data<jupyter_code>custom_unit_dict = {
         "gene_unit"          : "words",
         "variation_unit"     : "words",
         # text transformed to sentences attribute
         "doc_unit"           : "word_list",
         "doc_form"           : "text",
         "divide_document"    : "single_unit"
      }
%autoreload
import global_utils
gen_data = global_utils.GenerateDataset(x_train_df, corpus_wordidx)
x_train_34_T, x_train_34_G, x_train_34_V, x_train_34_C = gen_data.generate_data(custom_unit_dict, 
                                                                             has_class=True,
                                                                             add_start_end_tag=True)
del gen_data
print("Train data")
print("text",np.array(x_train_34_T).shape, x_train_34_T[0][:1])
print("gene",np.array(x_train_34_G).shape, x_train_34_G[0])
print("variation",np.array(x_train_34_V).shape, x_train_34_V[0])
print("classes",np.array(x_train_34_C).shape, x_train_34_C[0])
%autoreload
import global_utils
gen_data = global_utils.GenerateDataset(x_val_df, corpus_wordidx)
x_val_34_T, x_val_34_G, x_val_34_V, x_val_34_C = gen_data.generate_data(custom_unit_dict, 
                                                                             has_class=True,
                                                                             add_start_end_tag=True)
del gen_data
print("Val data")
print("text",np.array(x_val_34_T).shape, x_val_34_T[98][:1])
print("gene",np.array(x_val_34_G).shape, x_val_34_G[0])
print("variation",np.array(x_val_34_V).shape, x_val_34_V[0])
print("classes",np.array(x_val_34_C).shape, x_val_34_C[0])<jupyter_output>Val data
text (333,) [[352216, 252037, 156537, 91785, 67201, 109857, 123191, 209585, 213751, 5638, 0, 126280, 49123, 331220, 0, 352217]]
gene (333, 3) [352216, 217983, 352217]
variation (333,) [352216, 41934, 352217]
classes (333,) 4<jupyter_text>format data<jupyter_code>word_unknown_tag_idx   = corpus_wordidx["<UNK>"]
char_unknown_tag_idx   = global_utils.char_unknown_tag_idx
MAX_DOC_LEN = 500 # no of sentences in a document
MAX_SENT_LEN = 80 # no of words in a sentence
for doc_i, doc in enumerate(x_train_34_T):
    x_train_34_T[doc_i] = x_train_34_T[doc_i][:MAX_DOC_LEN]
    # padding sentences
    if len(x_train_34_T[doc_i]) < MAX_DOC_LEN:
        for not_used_i in range(0,MAX_DOC_LEN - len(x_train_34_T[doc_i])):
            x_train_34_T[doc_i].append([word_unknown_tag_idx]*MAX_SENT_LEN)
    # padding words
    x_train_34_T[doc_i] = pad_sequences(x_train_34_T[doc_i], maxlen=MAX_SENT_LEN, value=word_unknown_tag_idx)
    
for doc_i, doc in enumerate(x_val_34_T):
    x_val_34_T[doc_i] = x_val_34_T[doc_i][:MAX_DOC_LEN]
    # padding sentences
    if len(x_val_34_T[doc_i]) < MAX_DOC_LEN:
        for not_used_i in range(0,MAX_DOC_LEN - len(x_val_34_T[doc_i])):
            x_val_34_T[doc_i].append([word_unknown_tag_idx]*MAX_SENT_LEN)
    # padding words
    x_val_34_T[doc_i] = pad_sequences(x_val_34_T[doc_i], maxlen=MAX_SENT_LEN, value=word_unknown_tag_idx)
    
x_train_34_T = np.array(x_train_34_T)
x_val_34_T = np.array(x_val_34_T)
print(x_val_34_T.shape, x_train_34_T.shape)
x_train_34_G = pad_sequences(x_train_34_G, maxlen=MAX_SENT_LEN, value=word_unknown_tag_idx)
x_train_34_V = pad_sequences(x_train_34_V, maxlen=MAX_SENT_LEN, value=word_unknown_tag_idx)

x_val_34_G = pad_sequences(x_val_34_G, maxlen=MAX_SENT_LEN, value=word_unknown_tag_idx)
x_val_34_V = pad_sequences(x_val_34_V, maxlen=MAX_SENT_LEN, value=word_unknown_tag_idx)

print(x_train_34_G.shape, x_train_34_V.shape)
print(x_val_34_G.shape, x_val_34_V.shape)<jupyter_output>(2988, 80) (2988, 80)
(333, 80) (333, 80)<jupyter_text>keras np_utils.to_categorical expects zero index categorical variableshttps://github.com/fchollet/keras/issues/570<jupyter_code>x_train_34_C = np.array(x_train_34_C) - 1
x_val_34_C = np.array(x_val_34_C) - 1
x_train_34_C = np_utils.to_categorical(np.array(x_train_34_C), 9)
x_val_34_C = np_utils.to_categorical(np.array(x_val_34_C), 9)
print(x_train_34_C.shape, x_val_34_C.shape)<jupyter_output>(2988, 9) (333, 9)<jupyter_text>Need to form 3 dimensional target data for rationale model training<jupyter_code>temp = (x_train_34_C.shape[0],1,x_train_34_C.shape[1])
x_train_34_C_sent = np.repeat(x_train_34_C.reshape(temp[0],temp[1],temp[2]), MAX_DOC_LEN, axis=1)

#sentence test targets
temp = (x_val_34_C.shape[0],1,x_val_34_C.shape[1])
x_val_34_C_sent = np.repeat(x_val_34_C.reshape(temp[0],temp[1],temp[2]), MAX_DOC_LEN, axis=1)

print(x_train_34_C_sent.shape, x_val_34_C_sent.shape)<jupyter_output>(2988, 500, 9) (333, 500, 9)<jupyter_text>Embedding layer  for words<jupyter_code>WORD_EMB_SIZE = 200
%autoreload
import global_utils
ft_file_path = "/home/bicepjai/Projects/Deep-Survey-Text-Classification/data_prep/processed/stage1/pretrained_word_vectors/ft_sg_200d_50e.vec"
trained_embeddings = global_utils.get_embeddings_from_ft(ft_file_path, WORD_EMB_SIZE, corpus_vocab_list)
trained_embeddings.shape<jupyter_output><empty_output><jupyter_text>for characters<jupyter_code>CHAR_EMB_SIZE = 64
char_embeddings = np.random.randn(global_utils.CHAR_ALPHABETS_LEN, CHAR_EMB_SIZE)
char_embeddings.shape<jupyter_output><empty_output><jupyter_text>Models  prep<jupyter_code>%autoreload
import tensorflow.contrib.keras as keras
import tensorflow as tf

from keras import backend as K

from keras.engine import Layer, InputSpec, InputLayer

from keras.models import Model, Sequential

from keras.layers import Dropout, Embedding, concatenate
from keras.layers import Conv1D, MaxPool1D, Conv2D, MaxPool2D, ZeroPadding1D, GlobalMaxPool1D
from keras.layers import Dense, Input, Flatten, BatchNormalization
from keras.layers import Concatenate, Dot, Merge, Multiply, RepeatVector
from keras.layers import Bidirectional, TimeDistributed
from keras.layers import SimpleRNN, LSTM, GRU, Lambda, Permute

from keras.layers.core import Reshape, Activation
from keras.optimizers import Adam
from keras.callbacks import ModelCheckpoint,EarlyStopping,TensorBoard
from keras.constraints import maxnorm
from keras.regularizers import l2<jupyter_output><empty_output><jupyter_text>model_1: paper<jupyter_code>text_seq_input = Input(shape=(MAX_SENT_LEN,), dtype='int32')
text_embedding = Embedding(vocab_size, WORD_EMB_SIZE, input_length=MAX_SENT_LEN,
                            weights=[trained_embeddings], trainable=True)(text_seq_input)

model_1 = Sequential([
    Embedding(vocab_size, WORD_EMB_SIZE, weights=[trained_embeddings], 
               input_length=MAX_SENT_LEN, trainable=True),
    LSTM(32),
    Dense(9, activation='softmax')
])<jupyter_output><empty_output><jupyter_text>training<jupyter_code>model_1.compile(loss='categorical_crossentropy', optimizer=Adam(), metrics=['categorical_accuracy'])
model_1.summary()
%rm -rf ./tb_graphs/*
tb_callback = keras.callbacks.TensorBoard(log_dir='./tb_graphs', histogram_freq=0, write_graph=True, write_images=True)
checkpointer = ModelCheckpoint(filepath="model_1_weights.hdf5", 
                                    verbose=1,
                                    monitor="val_categorical_accuracy",
                                    save_best_only=True,
                                    mode="max")
with tf.Session() as sess:
    # model = keras.models.load_model('current_model.h5')
    sess.run(tf.global_variables_initializer())
    try:
        model_1.load_weights("model_1_weights.hdf5")
    except IOError as ioe:
        print("no checkpoints available !")
    model_1.fit(x_train_21_T, x_train_21_C, 
          validation_data=(x_val_21_T, x_val_21_C),
          epochs=5, batch_size=1024, shuffle=True,
          callbacks=[tb_callback,checkpointer])
    #model.save('current_sent_model.h5')<jupyter_output>no checkpoints available !
Train on 1086419 samples, validate on 128341 samples
Epoch 1/5
1085440/1086419 [============================>.] - ETA: 0s - loss: 1.4540 - categorical_accuracy: 0.4674Epoch 00000: val_categorical_accuracy improved from -inf to 0.53659, saving model to model_1_weights.hdf5
1086419/1086419 [==============================] - 126s - loss: 1.4537 - categorical_accuracy: 0.4675 - val_loss: 1.2638 - val_categorical_accuracy: 0.5366
Epoch 2/5
1085440/1086419 [============================>.] - ETA: 0s - loss: 1.0050 - categorical_accuracy: 0.6341Epoch 00001: val_categorical_accuracy improved from 0.53659 to 0.57913, saving model to model_1_weights.hdf5
1086419/1086419 [==============================] - 130s - loss: 1.0049 - categorical_accuracy: 0.6341 - val_loss: 1.1991 - val_categorical_accuracy: 0.5791
Epoch 3/5
1085440/1086419 [============================>.] - ETA: 0s - loss: 0.8365 - categorical_accuracy: 0.6792Epoch 00002: val_categorical_accuracy improved from[...]<jupyter_text>model_2: with GRU<jupyter_code>text_seq_input = Input(shape=(MAX_SENT_LEN,), dtype='int32')
text_embedding = Embedding(vocab_size, WORD_EMB_SIZE, input_length=MAX_SENT_LEN,
                            weights=[trained_embeddings], trainable=True)(text_seq_input)

model_2 = Sequential([
    Embedding(vocab_size, WORD_EMB_SIZE, weights=[trained_embeddings], 
               input_length=MAX_SENT_LEN, trainable=True),
    GRU(32),
    Dense(9, activation='softmax')
])<jupyter_output><empty_output><jupyter_text>training<jupyter_code>model_2.compile(loss='categorical_crossentropy', optimizer=Adam(), metrics=['categorical_accuracy'])
model_2.summary()
%rm -rf ./tb_graphs/*
tb_callback = keras.callbacks.TensorBoard(log_dir='./tb_graphs', histogram_freq=0, write_graph=True, write_images=True)
checkpointer = ModelCheckpoint(filepath="model_2_weights.hdf5", 
                                    verbose=1,
                                    monitor="val_categorical_accuracy",
                                    save_best_only=True,
                                    mode="max")
with tf.Session() as sess:
    # model = keras.models.load_model('current_model.h5')
    sess.run(tf.global_variables_initializer())
    try:
        model_2.load_weights("model_2_weights.hdf5")
    except IOError as ioe:
        print("no checkpoints available !")
    model_2.fit(x_train_21_T, x_train_21_C, 
          validation_data=(x_val_21_T, x_val_21_C),
          epochs=5, batch_size=1024, shuffle=True,
          callbacks=[tb_callback,checkpointer])
    #model.save('current_sent_model.h5')<jupyter_output>no checkpoints available !
Train on 1086419 samples, validate on 128341 samples
Epoch 1/5
1085440/1086419 [============================>.] - ETA: 0s - loss: 1.2653 - categorical_accuracy: 0.5386Epoch 00000: val_categorical_accuracy improved from -inf to 0.57209, saving model to model_2_weights.hdf5
1086419/1086419 [==============================] - 221s - loss: 1.2651 - categorical_accuracy: 0.5386 - val_loss: 1.2130 - val_categorical_accuracy: 0.5721
Epoch 2/5
1085440/1086419 [============================>.] - ETA: 0s - loss: 0.8859 - categorical_accuracy: 0.6618Epoch 00001: val_categorical_accuracy improved from 0.57209 to 0.58246, saving model to model_2_weights.hdf5
1086419/1086419 [==============================] - 222s - loss: 0.8859 - categorical_accuracy: 0.6618 - val_loss: 1.2238 - val_categorical_accuracy: 0.5825
Epoch 3/5
1085440/1086419 [============================>.] - ETA: 0s - loss: 0.7810 - categorical_accuracy: 0.6904Epoch 00002: val_categorical_accuracy improved from[...]
{'fraction_non_alphanumeric': 0.12130975874984896, 'fraction_numerical': 0.07865802086270088, 'mean_word_length': 4.023265223548452, 'pattern_counts': {'":': 2, '<': 86, '<?xml version=': 0, '>': 90, 'https://': 4, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 100, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '24', 'dir': 'jupyter-structured-clean-dedup', 'id': '15612', 'license': 'MIT', 'n_tokens_mistral': 10877, 'n_tokens_neox': 9372, 'n_words': 1536, 'path': 'deep_models/paper_09_dse_lstm/models.ipynb', 'repo_name': 'bicepjai/Deep-Survey-on-Text-Classification', 'size': '24829'}