#!/usr/bin/python3
# *-* coding: utf-8 *-*
'''
Created on 19 de Nov. de 2016

@author: <NAME>
'''

import logging
import re
import tensorflow as tf
import numpy as np


from model.abstract_classification_strategy import AbstractClassificationStrategy
from model.factory_meta_corpus import FactoryMetaCorpus
from model.utils.Errors import NonWordEmbeddingsHandleType
from model.utils.Errors import NonMetaCorpusDefined
from model.utils.semantic_relations_corpus_altlex import SemanticRelationsCorpusAltLex
from sklearn.metrics.classification import precision_score, recall_score,\
    f1_score
from model.factory_backprogation_optimizer import FactoryBackpropagationOptimizer
from model.utils.backprogragation_optimizers_creator_method_name import BackpropagationOptimizersCreatorMethodName
from model.factory_padding_strategies import FactoryPaddingStrategies
from model.utils.padding_strategies_name import PaddingStrategiesName

class AltLexSentenceEncodingLexicalMarkerInSecondArgumentLSTM(AbstractClassificationStrategy):
    '''It is a specific implementation of AbstractClassificationStrategy
    
    
    Attributes:
        __meta_corpus: The information built from the training corpus
        
        __meta_corpus_type: The specific implementation of the AbstractCorpus
        class
        
        __nlp_processing: This is the class that implements all the nlp methods
        
        __model_session: It is the tensorFlow session.
    
    '''


    def __init__(self):
        '''
        Constructor
        '''
        self.__training_corpus = None
        self.__dev_corpus = None
        self.__test_corpus = None
        self.__meta_corpus = None
        self.__meta_corpus_type = None
        self.__nlp_processing = None
        self.__word_embedding_handle = None
        self.__max_vocabulary_size = None
        self.__max_length_text_one = None
        self.__max_length_lexical_marker = None
        self.__max_length_text_two = None
        self.__model_session = None
        self.__f_loss_before = None
        self.__f_loss = None
        self.__accuracy = None
        self.__prediction_labels_values = None
        self.__evaluation_accuracy_value = None
        self.__evaluation_loss_value = None
        self.__evaluation_accuracy_value_best_model = None
        self.__evaluation_loss_value_best_model = None
        self.__evaluation_precision_per_class = None
        self.__evaluation_precision_per_class_best_model = None
        self.__evaluation_recall_per_class = None
        self.__evaluation_recall_per_class_best_model = None
        self.__evaluation_f1_per_class = None
        self.__evaluation_f1_per_class_best_model = None
        self.__evaluation_precision_general = None 
        self.__evaluation_precision_general_best_model = None
        self.__evaluation_recall_general = None
        self.__evaluation_recall_general_best_model = None
        self.__evaluation_f1_general = None
        self.__evaluation_f1_general_best_model = None
        self.__best_epoch = None
        self.__best_batch = None
        self.__nn_envirionment_log_dir = None
        self.__nn_envirionment_model_dir = None
        self.__tf_train_writer = None
        self.__random_seed = None
        
        #Model
        self.__text_one_lstm_cell_fw = None
        self.__text_two_lstm_cell_fw = None
        
        #Training
        self.__number_of_epochs = None
        self.__size_of_batches = None
        self.__steps_to_monitor = None
        
        #NN placeholders
        self.__x_text_one_placeholder = None
        self.__x_text_two_placeholder = None
        self.__embeddings_weight_placeholder = None
        self.__y_real_labels_placeholder = None
        self.__train_summary_average_loss_function_placeholder = None
        self.__train_summary_average_accuracy_placeholder = None

        
        self.__train_step = None
        
        #Optimizer
        self.__optimiezer = None
        
        #Padding strategy
        self.__padding_strategy = None
        
        #Dropout properties
        
        self.__drop_out_training_keep_prob = None
        self.__drop_out_training_text_one_placeholder = None
        self.__drop_out_training_text_two_placeholder = None
        self.__drop_out_test_keep_prob = None
        
        #Regularization
        self.__landa_regularization = None
        self.__landa_loss_regularization = None
        
        #Number of layers
        self.__number_coding_dense_layers = None
        
        #Summary properties
        self.__train_summary_average_accuracy = None
        self.__train_summary_average_loss = None
        self.__dev_summary_loss_function = None
        self.__dev_summary_accuracy = None
        self.__train_histogram_encoding_text_one = None
        self.__train_histogram_encoding_text_two = None
        self.__dev_histogram_encoding_text_one = None
        self.__dev_histogram_encoding_text_two = None
        
        
        #Early stopping
        self.__checkpoint_saver = None
        
        
        #Files
        self.__progress_file = None
        
        
        
        
    @property
    def meta_training_corpus(self):
        """Getter method for the meta_training_corpus property
        
        Returns:
            The object of the meta_corpus of the training set.
        """
        ...
        
    @property
    def meta_dev_corpus(self):
        """Getter method for the meta_dev_corpus
        
        Returns:
            The object of the meta_corpus of the development set.
        """
        return(self.__meta_dev_corpus)
        
    @property
    def meta_test_corpus(self):
        """Getter method for the meta_dev_corpus
        
        Returns:
            The object of the meta_corpus of the test set.
        """
        return(self.__meta_test_corpus)
        
    @property
    def meta_corpus_type(self):
        """Getter method for the type of the meta_corpus
        
        Returns:
            Return the specific implementation of the meta_corpus abstract calss
        """
        return(self.__meta_corpus_type)
        
    @property
    def meta_corpus(self):
        """Getter method for the meta_corpus class
        
        Returns:
            Returns the meta corpus specific implementation
        """
        return(self.__meta_corpus)
    
    @property
    def nlp_processing(self):
        """The getter method for the class that implements the specific methods
        related to nlp operations
        """
        ...
        
    @property
    def evaluation_accuracy_value(self):
        """Getter method of the property evaluation_accuracy_value
        """
        return(self.__evaluation_accuracy_value)
        
    @property
    def evaluation_loss_value(self):
        """Getter method of the property evaluation_accuracy_value
        """
        return(self.__evaluation_loss_value)
    
    @property
    def steps_to_monitor(self):
        """Getter method for the property steps_to_monitor
        
        Number of training steps to run in order to evaluate the model.
        
        Returns:
            A integer with the number of steps to run before evaluating the 
            training.
        """
        return(self.__steps_to_monitor)
    
    @steps_to_monitor.setter
    def steps_to_monitor(self, a_steps_to_monitor):
        """Setter method for the property steps_to_monitor
        
        Args:
            a_steps_to_monitor: int value with the number of steps to run 
            before the evaluation of the training.
        """
        self.__steps_to_monitor = a_steps_to_monitor
        
        
    @property
    def nn_envirionment_model_dir(self):
        """Getter method for the property nn_environment_model_dir
        
        Returns:
            A string with the path of the environment model dir.
        """
        return(self.__nn_envirionment_model_dir)
    
    @nn_envirionment_model_dir.setter
    def nn_envirionment_model_dir(self, a_nn_environment_model_dir):
        """Getter method for the property nn_environment_model_dir
        
        Args:
            a_nn_environment_model_dir: A string with the value of the path to
            the environment model dir
        """
        self.__nn_envirionment_model_dir = a_nn_environment_model_dir
    
    
    @property
    def evaluation_accuracy_value_best_model(self):
        """Getter method of the property evaluation_accuracy_value_best_model
        
        Returns:
            A float value which represents the accuracy of the best model of the
            training.
        """
        return(self.__evaluation_accuracy_value_best_model)
        
    @property
    def evaluation_loss_value_best_model(self):
        """Getter method of the property evaluation_loss_value_best_model
        
        Returns:
            A float value which represents the loss of the best model of the
            training.
        """
        return(self.__evaluation_loss_value_best_model)
    
    @property
    def best_epoch(self):
        """Getter method for the property best_epoch
        
        Returns:
            Int value with the best epoch of the model
        """
        return(self.__best_epoch)
    
    @property
    def best_batch(self):
        """Getter method for the property best_batch
        
        Returns:
            Int value with the best batch of the model
        """
        return(self.__best_batch)
    
    
    @property
    def max_length_text_one(self):
        """Getter method for the property max_length_text_one
        
        Returns:
            The value of the maximum length of the text before the lexical marker 
        """
        return(self.__max_length_text_one)
    
    @property
    def max_length_text_two(self):
        """Getter method for the property max_length_text_two
        
        Returns:
            The value of the maximum length of the text after the lexical marker 
        """
        return(self.__max_length_text_two)
    
    @property
    def max_length_lexical_marker(self):
        """Getter method for the property max_length_lexical_marker
        
        Returns:
            The value of the maximum length of the lexical marker
        """
        return(self.__max_length_lexical_marker)
    
    @property
    def evaluation_precision_per_class(self):
        """Getter method for the property evaluation_precision_per_class
        
        Returns:
            A list of floats with the value of the precision of each class. The
            index matches with the value of the class in SemanticRelationsCorpusAltlex 
        """
        return(self.__evaluation_precision_per_class)
        
    @property
    def evaluation_recall_per_class(self):
        """Getter method for the property evaluation_recall_per_class
        
        Returns:
            A list of floats with the value of the recall of each class. The
            index matches with the value of the class in SemanticRelationsCorpusAltlex 
        """
        return(self.__evaluation_recall_per_class)
    
    @property
    def evaluation_f1_per_class(self):
        """Getter method for the property evaluation_f1_per_class
        
        Returns:
            A list of floats with the value of the f1 of each class. The
            index matches with the value of the class in SemanticRelationsCorpusAltlex 
        """
        return(self.__evaluation_f1_per_class)
        
    @property
    def evaluation_precision_general(self):
        """Getter method for the property evaluation_precision_general
        
        Returns:
            A float with the general precision score of the method 
        """
        return(self.__evaluation_precision_general)
    
    @property
    def evaluation_recall_general(self):
        """Getter method for the property evaluation_recall_general
        
        Returns:
            A float with the general recall score of the method 
        """
        return(self.__evaluation_recall_general)
        
    @property
    def evaluation_f1_general(self):
        """Getter method for the property evaluation_f1_general
        
        Returns:
            A float with the general f1 score of the method 
        """
        return(self.__evaluation_f1_general)

    
    @property
    def evaluation_precision_per_class_best_model(self):
        """Getter method for the property evaluation_precision_per_class_best_model
        
        Returns:
            A list of float values with the precision score of the best model per class.
        """
        return(self.__evaluation_precision_per_class_best_model)
    
    @property
    def evaluation_recall_per_class_best_model(self):
        """Getter method for the property evaluation_recall_per_class_best_model
        
        Returns:
            A list of float values with the recall score of the best model per class.
        """
        return(self.__evaluation_recall_per_class_best_model)
    
    @property
    def evaluation_f1_per_class_best_model(self):
        """Getter method for the property evaluation_f1_per_class_best_model
        
        Returns:
            A list of float values with the f1 score of the best model per class.
        """
        return(self.__evaluation_f1_per_class_best_model)
    
    @property
    def evaluation_precision_general_best_model(self):
        """Getter method for the property evaluation_precision_general_best_model
        
        Returns:
            A float value with the precision score of the best model.
        """
        return(self.__evaluation_precision_general_best_model)
    
    @property
    def evaluation_recall_general_best_model(self):
        """Getter method for the property evaluation_precision_general_best_model
        
        Returns:
            A float value with the precision score of the best model.
        """
        return(self.__evaluation_recall_general_best_model)
    
    @property
    def evaluation_f1_general_best_model(self):
        """Getter method for the property evaluation_precision_general_best_model
        
        Returns:
            A float value with the precision score of the best model.
        """
        return(self.__evaluation_f1_general_best_model)
    
    
    @property
    def number_coding_dense_layers(self):
        """Getter method for the property number_coding_dense_layers
        
        Returns:
            An integer which represents the number of intermediate layers to be
            run upon the data.
        """
        return(self.__number_coding_dense_layers)
    
    
    def initialize(self, **args):
        """This method initializes and set additional parameters to the model
        """
        
        logging.debug("ALTLEX_SENTENCE_ENCODING_LSTM: Starting the initialization of class")
        
        if('METACORPUSTYPE' not in args):
            raise NonMetaCorpusDefined("ALTLEX_SENTENCE_ENCODING_LSTM: Any strategy to manage the meta-data. You must define an implementation of a AbstractMetaExpData")
        
        
        if(args['METACORPUSTYPE'] is None):
            raise NonMetaCorpusDefined("ALTLEX_SENTENCE_ENCODING_LSTM: The MetaExpData strategy is None. Please, create one, I can't work with None values.")
        
        self.__meta_corpus_type = args['METACORPUSTYPE']
        
        
        self.__training_corpus = args['TRAININGCORPUS']
        self.__meta_corpus = FactoryMetaCorpus.creator(self.__meta_corpus_type)
        self.__meta_corpus.corpus_train = self.__training_corpus
        
        
        if('TESTCORPUS' in args):
            self.__test_corpus = args['TESTCORPUS']
            
        if(self.__test_corpus is not None):
            self.__meta_corpus.corpus_test = self.__test_corpus
        
        if("DEVCORPUS" in args):
            self.__dev_corpus = args['DEVCORPUS']
            
        if(self.__dev_corpus is not None):
            self.__meta_corpus.corpus_dev = self.__dev_corpus
        
        self.__word_embedding_handle = args['WORDEMBEDDINGHANDLE']
        if(self.__word_embedding_handle == None):
            raise NonWordEmbeddingsHandleType("ALTLEX_SENTENCE_ENCODING_LSTM: No word embeddings handle defined")
        
        self.__nlp_processing = args['NLPPROCESSINGTOOL']
        self.__meta_corpus.nlp_processing_tool = self.__nlp_processing
        
        random_seed = args['RANDOMSEED']
        self.__random_seed = random_seed
        
        if ('MAXVOCABULARYSIZE' in args):
            self.__max_vocabulary_size = args['MAXVOCABULARYSIZE']
        
        if("SIZEOFBATCHES" in args):
            if(args["SIZEOFBATCHES"] is None):
                self.__size_of_batches = 100
            else:
                self.__size_of_batches = args["SIZEOFBATCHES"]
        else:
            self.__size_of_batches = 100
        
        if ("NUMBEROFEPOCHS" in args):
            if(args["NUMBEROFEPOCHS"] is None):
                self.__number_of_epochs = 2
            else:
                self.__number_of_epochs = args["NUMBEROFEPOCHS"]
        else:
            self.__number_of_epochs = 2
        
        if("STEPSTOMONITOR" in args):
            self.__steps_to_monitor = args["STEPSTOMONITOR"]
        
        
        
        if ("NNENVIRONMENTLOGDIR" in args):
            self.__nn_envirionment_log_dir = args["NNENVIRONMENTLOGDIR"]
            
        if("NNENVIRONMENTMODELDIR" in args):
            self.__nn_envirionment_model_dir = args["NNENVIRONMENTMODELDIR"]
            
            
        if ("DROPOUTTRAININGVALUE" in args):
            self.__drop_out_training_keep_prob = args["DROPOUTTRAININGVALUE"]
            
        if("DROPOUTTESTVALUE" in args):
            self.__drop_out_test_keep_prob = args["DROPOUTTESTVALUE"]
            
        if("LANDAREGULARIZATION" in args):
            self.__landa_regularization = args["LANDAREGULARIZATION"]
            
        if("LANDALOSSREGULARIZATION" in args):
                self.__landa_loss_regularization = args["LANDALOSSREGULARIZATION"]
        
        if("PATHPROGRESSFILE" in args):
            if(args["PATHPROGRESSFILE"] is not None):
                self.__progress_file = open(args["PATHPROGRESSFILE"], "w")
        
        
        if("BACKPROPAGATIONOPTIMIZER" in args):
            if(args["BACKPROPAGATIONOPTIMIZER"] is not None):
                self.__optimiezer = FactoryBackpropagationOptimizer.creator(args["BACKPROPAGATIONOPTIMIZER"])
            else:
                self.__optimiezer = FactoryBackpropagationOptimizer.creator(BackpropagationOptimizersCreatorMethodName.ADADELTA.name)
                
                
                
        if("PADDINGSTRATEGY" in args):
            if(args["PADDINGSTRATEGY"] is not None):
                self.__padding_strategy = FactoryPaddingStrategies.creator(args["PADDINGSTRATEGY"])
            else:
                self.__padding_strategy = FactoryPaddingStrategies.creator(PaddingStrategiesName.PADDING_MAXIMUM.name)
        
        if("NUMBEROFDENSELAYERS" in args):
                if("NUMBEROFDENSELAYERS" is not None):
                    self.__number_coding_dense_layers = args["NUMBEROFDENSELAYERS"]
                else:
                    self.__number_coding_dense_layers = 3
        
        
        logging.debug("ALTLEX_SENTENCE_ENCODING_LSTM: Ending the initialization of class")
    
    
    
    def __weight_matrix_initialization(self, shape, random_seed):
        """Initialize the weights and bias variable.
        
        Args:
            shape: The shape of the variables,
        """
#         return(tf.truncated_normal(shape, stddev=0.1, seed=random_seed))
        return(tf.contrib.layers.xavier_initializer(seed=random_seed))
    
    
    def __bias_vector_initialization(self, shape, constant_value, type_of_values=None):
        
        """Initialization method for the bias vector
        
        Args:
            shape: A list with  the shape of the vector
        """
        return(tf.constant(constant_value, dtype=type_of_values, shape=shape))
    
    
#     def __strategy_lenght_documents(self):
#         """Method for the definition of the maximum length of the documents.
#         """
#         self.__max_length_text_one = int(np.max(self.__meta_corpus.length_documents[0]))
#         self.__max_length_text_two = int(np.max(self.__meta_corpus.length_documents[1]))
#         self.__max_length_lexical_marker = (int(np.max(self.__meta_corpus.length_documents[2]))) + 2 #Two special chars before/after the lexical marker
        
    
    def compile_model(self):
        """This method compiles the model
        """
        
        logging.debug("ALTLEX_SENTENCE_ENCODING_LSTM: Starting the compilation of the architecture of the neural network")
        
        #Build features
        self.__meta_corpus.build_documents_features(self.__word_embedding_handle, max_frequent_features=self.__max_vocabulary_size)
        
        length_of_vectors = [self.__meta_corpus.length_documents[0], self.__meta_corpus.length_documents[1], self.__meta_corpus.length_documents[2]]
        
        self.__max_length_text_one, self.__max_length_text_two, self.__max_length_lexical_marker = self.__padding_strategy.length_padding_calcuation(length_of_vectors)
        #Set random seed
        
        tf.set_random_seed(self.__random_seed)
        
        #Regularizer
        weights_bias_regularizer = tf.contrib.layers.l2_regularizer(self.__landa_regularization)
        
        #Placeholders
        self.__x_text_one_placeholder = None
        self.__x_text_two_placeholder = None
        
        
        logging.debug("ALTLEX_SENTENCE_ENCODING_LSTM: Definition of the placeholder for the text previous the lexical marker")
        with tf.variable_scope("input_text_one_placeholder"):
            self.__x_text_one_placeholder = tf.placeholder(tf.int32, [None, self.__max_length_text_one], name="ph_text_one_input")
        
        logging.debug("ALTLEX_SENTENCE_ENCODING_LSTM: Definition of the placeholders for the text after the lexical marker")
        with tf.variable_scope("input_text_two_layer"):
            self.__x_text_two_placeholder = tf.placeholder(tf.int32, [None, self.__max_length_text_two + self.__max_length_lexical_marker + 1], name="ph_text_two_input")
        
        #Placeholders summary
        self.__train_summary_average_accuracy_placeholder = tf.placeholder(tf.float32, name="ph_average_accuracy")
        self.__train_summary_average_loss_function_placeholder = tf.placeholder(tf.float32, name="ph_average_loss")

        #Dropout definition
        if(self.__drop_out_training_keep_prob is not None):
            logging.debug("ALTLEX_SENTENCE_ENCODING_LSTM: Definition of the dropout layers (premise and hypothesis)")
            with tf.variable_scope("dropout_text_one_layer"):
                self.__drop_out_training_text_one_placeholder = tf.placeholder(tf.float32, name="dropout_text_one_layer")
            with tf.variable_scope("dropout_text_two_layer"):
                self.__drop_out_training_text_two_placeholder = tf.placeholder(tf.float32, name="dropout_text_two_layer")
        
        #Layer to reduce embeddings  dimensions
        logging.debug("ALTLEX_SENTENCE_ENCODING_LSTM: Definition of the embedding layer")
        embeddings_weight = None
        with tf.variable_scope("embeddings_layer"):
            embeddings_initialization = tf.constant_initializer(np.array(self.__word_embedding_handle.word_embeddings))
            embeddings_shape = [self.__word_embedding_handle.number_of_embedding_vectors(), self.__word_embedding_handle.size_embedding_vector()]
            embeddings_weight = tf.get_variable("var_embeddings_weights", initializer=embeddings_initialization, shape=embeddings_shape, trainable=True)
        
        #Encoding layer
        logging.debug("ALTLEX_SENTENCE_ENCODING_LSTM: premise represented by word embeddings vectors")
        with tf.variable_scope("embeddings_lookup_text_one_layer"):
            x_text_one_embeddings = tf.nn.embedding_lookup(embeddings_weight, self.__x_text_one_placeholder, partition_strategy="mod", name="op_x_text_one_embeddings_lookup")
            
        with tf.variable_scope("encoding_lstm_text_one_cell_fw"):
            with tf.variable_scope("lstm_cell"):
                self.__text_one_lstm_cell_fw = tf.nn.rnn_cell.LSTMCell(self.__word_embedding_handle.size_embedding_vector(), use_peepholes=False, state_is_tuple=True)
                x_text_one_encoding, _ = tf.nn.dynamic_rnn(self.__text_one_lstm_cell_fw, inputs=x_text_one_embeddings, dtype=tf.float32, time_major=False)
                self.__train_histogram_encoding_text_one = tf.histogram_summary(x_text_one_encoding.op.name, x_text_one_encoding, name=x_text_one_encoding.op.name + "_histogram")
                self.__dev_histogram_encoding_text_one = tf.histogram_summary(x_text_one_encoding.op.name + "_dev", x_text_one_encoding, name="dev_" + x_text_one_encoding.op.name + "_histogram") 
            if(self.__drop_out_training_keep_prob is not None):
                x_text_one_encoding = tf.nn.dropout(x_text_one_encoding, self.__drop_out_training_text_one_placeholder, seed=self.__random_seed, name="dropout_input_text_one_encoding_layer_after")
                
                
        logging.debug("ALTLEX_SENTENCE_ENCODING_LSTM: hypothesis represented by word embeddings vectors")
        with tf.variable_scope("embeddings_lookup_text_two_layer"):
            x_text_two_embeddings = tf.nn.embedding_lookup(embeddings_weight, self.__x_text_two_placeholder, partition_strategy="mod", name="op_x_text_two_embeddings")

        with tf.variable_scope("encoding_lstm_text_two_cell_fw"):
            with tf.variable_scope("lstm_cell"):
                self.__text_two_lstm_cell_fw = tf.nn.rnn_cell.LSTMCell(self.__word_embedding_handle.size_embedding_vector(), use_peepholes=False, state_is_tuple=True)
                x_text_two_encoding, _ = tf.nn.dynamic_rnn(self.__text_two_lstm_cell_fw, inputs=x_text_two_embeddings, dtype=tf.float32, time_major=False)
                self.__train_histogram_encoding_text_two = tf.histogram_summary(x_text_two_encoding.op.name, x_text_two_encoding, name=x_text_two_encoding.op.name + "/histogram")
                self.__dev_histogram_encoding_text_two = tf.histogram_summary(x_text_two_encoding.op.name + "_dev", x_text_two_encoding, name="dev_" + x_text_two_encoding.op.name + "/histogram")
                if(self.__drop_out_training_keep_prob is not None):
                    x_text_two_encoding = tf.nn.dropout(x_text_two_encoding, self.__drop_out_training_text_two_placeholder, seed=self.__random_seed, name="dropout_input_text_two_encoding_layer_after")
            
        #Reduction dimension layer
        logging.debug("ALTLEX_SENTENCE_ENCODING_LSTM: encoding and reduction of the premise")
        
        
        x_text_one_encoded_reduced = None
        with tf.variable_scope("reduction_dimension_premise_layer"):
            #Reshaping of premise tensor (3d)
            new_shape_x_text_one = [-1, x_text_one_encoding.get_shape()[1].value * x_text_one_encoding.get_shape()[2].value]
            x_text_one_encoded_reduced = tf.reshape(x_text_one_encoding, shape=new_shape_x_text_one, name="reshaping_text_one_for_encoding_reduction")
            weights_shape = [x_text_one_encoded_reduced.get_shape()[1].value, 100]
            weights_initializer = self.__weight_matrix_initialization(weights_shape, self.__random_seed)
            bias_initializer = self.__bias_vector_initialization([100], 0.1) 
            x_text_one_weights_to_reduce = tf.get_variable("v_weights_to_reduce_shape_text_one", shape=weights_shape, initializer=weights_initializer, regularizer=weights_bias_regularizer)
            x_text_one_bias_to_reduce = tf.get_variable("v_bias_to_reduce_shape_text_one", initializer=bias_initializer)
            x_text_one_encoded_reduced = tf.tanh(tf.matmul(x_text_one_encoded_reduced, x_text_one_weights_to_reduce) + x_text_one_bias_to_reduce, name="text_one_reduction_tanh_layer")
            
        logging.debug("ALTLEX_SENTENCE_ENCODING_LSTM: encoding and reduction of the hypothesis")
        
        x_text_two_encoded_reduced = None
        with tf.variable_scope("reduction_dimension_text_two_layer"):
            #Reshaping of hypothesis tensor (3d)
            new_shape_x_text_two = [-1, x_text_two_encoding.get_shape()[1].value * x_text_two_encoding.get_shape()[2].value]
            x_text_two_encoded_reduced = tf.reshape(x_text_two_encoding, shape=new_shape_x_text_two, name="reshaping_text_two_for_encoding_reduction")
            weights_shape = [x_text_two_encoded_reduced.get_shape()[1].value, 100]
            weights_initializer = self.__weight_matrix_initialization(weights_shape, self.__random_seed)
            bias_initializer = self.__bias_vector_initialization([100], 0.1) 
            x_text_two_weights_to_reduce = tf.get_variable("v_weights_to_reduce_shape_text_two", shape=weights_shape, initializer=weights_initializer, regularizer=weights_bias_regularizer)
            x_text_two_bias_to_reduce = tf.get_variable("v_bias_to_reduce_shape_text_two", initializer=bias_initializer)
            x_text_two_encoded_reduced = tf.tanh(tf.matmul(x_text_two_encoded_reduced, x_text_two_weights_to_reduce) + x_text_two_bias_to_reduce, name="text_two_tanh_layer")
        
        #Concatenating sentence representation
        x_text_one_two_encoded = tf.concat(1, [x_text_one_encoded_reduced, x_text_two_encoded_reduced], name="concatenation_text_one_text_two")
        logging.debug("ALTLEX_SENTENCE_ENCODING_LSTM: concatenation of the premise and the hypothesis")
        
        x_text_one_two_fedfordwardtanh = x_text_one_two_encoded
        if(self.__number_coding_dense_layers != 0):
            with tf.variable_scope("fedforward_tanh_layer"):
                logging.debug("ALTLEX_SENTENCE_ENCODING_LSTM: definition of layers of feed forward network")
                weights_shape = [x_text_one_two_fedfordwardtanh.get_shape()[1].value, x_text_one_two_fedfordwardtanh.get_shape()[1].value]
                weights_initializer = self.__weight_matrix_initialization(weights_shape, self.__random_seed)
                bias_initializer = self.__bias_vector_initialization([x_text_one_two_fedfordwardtanh.get_shape()[1]], 0.1)
                weights_matrix = tf.get_variable("feedforward_tanh_layer_weights", shape=weights_shape, initializer=weights_initializer, regularizer=weights_bias_regularizer)
                bias_vector = tf.get_variable("fedforward_tanh_layer_bias", initializer=bias_initializer)
                 
                for layer in range(self.__number_coding_dense_layers):
                    logging.debug("ALTLEX_SENTENCE_ENCODING_LSTM: definition of layer %i of feed forward network"%layer)
                    x_text_one_two_fedfordwardtanh = tf.tanh(tf.matmul(x_text_one_two_fedfordwardtanh, weights_matrix) + bias_vector, name="fedfordward_layer_%i"%layer)
                
        
        self.__y_real_labels_placeholder = None
        with tf.name_scope("output_layer"):
            logging.debug("ALTLEX_SENTENCE_ENCODING_LSTM: definition of output layer")
            self.__y_real_labels_placeholder = tf.placeholder(tf.int32, [None], "ph_real_labels")
        
        
        y_classified = None
        with tf.variable_scope("classification_layer"):
            logging.debug("ALTLEX_SENTENCE_ENCODING_LSTM: definition of classification (softmax) layer")
            weights_shape = [x_text_one_two_fedfordwardtanh.get_shape()[1].value, len(SemanticRelationsCorpusAltLex.__members__)]
            weights_initializer = self.__weight_matrix_initialization(weights_shape, self.__random_seed) 
            bias_initializer = self.__bias_vector_initialization([len(SemanticRelationsCorpusAltLex.__members__)], 0.1) 
            
            weights_matrix_classification = tf.get_variable("classification_weights", shape=weights_shape, initializer=weights_initializer, regularizer=weights_bias_regularizer)
            bias_vector_classification = tf.get_variable("classification_bias", initializer=bias_initializer)
            y_classified_logits = tf.matmul(x_text_one_two_fedfordwardtanh, weights_matrix_classification) + bias_vector_classification
            y_classified = tf.nn.sparse_softmax_cross_entropy_with_logits(y_classified_logits, self.__y_real_labels_placeholder, name="classficiation_layer_softmax")
#             self.__train_summary_hystogram_classification_layer_without_softmax = tf.histogram_summary(y_classified_logits.op.name, tf.nn.softmax(y_classified_logits), name="train_summary_classification_layer_without_softmax")
            
        with tf.variable_scope("loss_function"):
            logging.debug("ALTLEX_SENTENCE_ENCODING_LSTM: definition of loss function.")
            
            self.__f_loss_before = tf.reduce_mean(y_classified, name="loss_calculation")
            #Application of regularization
            reg_losses = tf.get_collection(tf.GraphKeys.REGULARIZATION_LOSSES)
            if(len(reg_losses) == 0):
                self.__f_loss = self.__f_loss_before 
            else:
                self.__f_loss = self.__f_loss_before + (self.__landa_loss_regularization * tf.add_n(reg_losses))
                
            self.__train_summary_average_loss = tf.scalar_summary(self.__f_loss.op.name, self.__train_summary_average_loss_function_placeholder, name="train_summary_loss_function")
            self.__dev_summary_loss_function = tf.scalar_summary("dev_" + self.__f_loss.op.name, self.__f_loss, name="dev_summary_loss_function")
        
        
        self.__train_step = self.__optimiezer.minimize(self.__f_loss)
        
    
        with tf.variable_scope("accuracy"):
            logging.debug("ALTLEX_SENTENCE_ENCODING_LSTM: definition of accuracy")
            self.__prediction_labels_values = tf.arg_max(y_classified_logits, 1)
            correct_predictions = tf.equal(self.__prediction_labels_values, tf.to_int64(self.__y_real_labels_placeholder))
            self.__accuracy = tf.reduce_mean(tf.cast(correct_predictions, tf.float32, "accuracy"))
            self.__train_summary_average_accuracy = tf.scalar_summary(self.__accuracy.op.name, self.__train_summary_average_accuracy_placeholder, name="train_summary_accuracy")
            self.__dev_summary_accuracy = tf.scalar_summary("dev_" + self.__accuracy.op.name, self.__accuracy, name="dev_summary_accuracy")
            
        logging.debug("ALTLEX_SENTENCE_ENCODING_LSTM: Ending the compilation of the architecture of the neural network")
        
        
    
    def run_model(self):
        """Run the classification model
        
        Requires:
            A training and a test corpora
        """
        
        
#         merged_summaries = tf.merge_all_summaries()
        
        self.__model_session = tf.Session()        
        self.__model_session.run(tf.initialize_all_variables())
        
        
        if(self.__nn_envirionment_log_dir is not None):
            self.__tf_train_writer = tf.train.SummaryWriter(self.__nn_envirionment_log_dir, self.__model_session.graph)
        
        doc_ids = self.__training_corpus.get_corpus_ids()
        
        number_of_batches = int(self.__training_corpus.number_of_documents() / self.__size_of_batches)
        step = 0
        train_fetches = {"train_step":self.__train_step,
                                     "accuracy":self.__accuracy,
                                     "f_loss":self.__f_loss,
                                     "train_summary_loss": self.__train_summary_loss_function,
                                     "train_summary_accuracy": self.__train_summary_accuracy,
#                                      "train_summary_hystogram_classification_layer_without_softmax": self.__train_summary_hystogram_classification_layer_without_softmax}
                        }
        train_fetches_values = None
        
        
        for epoch in range(self.__number_of_epochs):
            for batch in range(number_of_batches + 1):
                start_index = batch * self.__size_of_batches
                end_index = (batch + 1) * self.__size_of_batches
                batch_doc_ids = doc_ids[start_index:end_index]
                if(len(batch_doc_ids) != 0):
                    
                    if(len(batch_doc_ids) < self.__size_of_batches):
                        batch_doc_ids += doc_ids[0:(self.__size_of_batches - len(batch_doc_ids))]
                    
                    
                    
                    premise_input_matrix = []
                    hypothesis_input_matrix = []
                    y_real_labels = []
                    for id in batch_doc_ids:
                        doc = self.__training_corpus.get_document(id)
                        premise_features = doc.get_value_name_feature("EMB_INDEX_TEXT_ONE")
                        if (len(premise_features) < self.__meta_corpus.max_length_document[0]):
                            for i in range(self.__meta_corpus.max_length_document[0] - len(premise_features)):
                                premise_features.append(0)  #This should be in meta_corpus
                        premise_input_matrix.append(premise_features)
                        
                        hypothesis_features = doc.get_value_name_feature("EMB_INDEX_TEXT_TWO")
                        if (len(hypothesis_features) < self.__meta_corpus.max_length_document[1]):
                            for i in range(self.__meta_corpus.max_length_document[1] - len(hypothesis_features)):
                                hypothesis_features.append(0)
                        
                        hypothesis_input_matrix.append(hypothesis_features)
                        y_real_labels.append(doc.real_label)
                    
                    
                    
                    
                    
                    train_fetches_values  = self.__model_session.run(train_fetches, feed_dict={self.__x_premise_placeholder:premise_input_matrix, 
                                                                                                                         self.__x_hypothesis_placeholder:hypothesis_input_matrix, 
                                                                                                                         self.__embeddings_weight_placeholder:self.__word_embedding_handle.word_embeddings, 
                                                                                                                         self.__y_real_labels_placeholder:y_real_labels})
                    
                    
                    
                    if (step % int((self.__size_of_batches/3)) == 0):
                        if(self.__tf_train_writer is not None):
                            self.__tf_train_writer.add_summary(train_fetches_values["train_summary_accuracy"])
                            self.__tf_train_writer.add_summary(train_fetches_values["train_summary_loss"])
                            self.__tf_train_writer.add_summary(train_fetches_values["train_summary_hystogram_classification_layer_without_softmax"])
                            
                            
                        print("Epoch {:d}/{:d}, batch {:d}/{:d}: Accuracy {:f}; Loss: {:f}".format(epoch+1, self.__number_of_epochs, batch+1, number_of_batches, train_fetches_values["accuracy"], train_fetches_values["f_loss"]))
                    
                    step += 1
                

        
    def __non_training_input_model_preparation(self, set_of_documents):
        """Build the input matrix with non-training data (test/development)
        
        
        Returns:
            premise_matrix: A matrix with the input features of all the premises
                of the input data
            hypothesis_matrix: A matrix with the input features of all the hypothesis
                of the input data
                
            y_real_labels: A list with the sparse real labels of the input
                documents.
        """
        logging.info("ALTLEX_SENTENCE_ENCODING_LSTM: Beginning of the building of non training model")
        
        doc_ids = set_of_documents.get_corpus_ids()
        text_one_matrix = []
        text_two_matrix = []
        y_real_labels = []
        own_get_document = set_of_documents.get_document
        own_append_text_one_matrix = text_one_matrix.append
        own_append_text_two_matrix = text_two_matrix.append
        own_append_real_labels = y_real_labels.append
        max_length_text_two = self.__max_length_lexical_marker + self.__max_length_text_two + 1
        for doc_id in doc_ids:
            doc = own_get_document(doc_id)
            text_one_features = doc.get_value_name_feature("EMB_INDEX_TEXT_ONE")
            text_one_features = self.__padding_if_neccesary(text_one_features, self.__max_length_text_one,0)
            own_append_text_one_matrix(text_one_features)
            
            text_two_features = doc.get_value_name_feature("EMB_INDEX_CAUSAL_MARKER")
            text_two_features += [1] + doc.get_value_name_feature("EMB_INDEX_TEXT_TWO")
            text_two_features = self.__padding_if_neccesary(text_two_features, max_length_text_two, 0)
            own_append_text_two_matrix(text_two_features)
            
            own_append_real_labels(doc.real_label)
            
        logging.info("ALTLEX_SENTENCE_ENCODING_LSTM: Ending of the building of non training model")
        return(text_one_matrix, text_two_matrix, y_real_labels)
    
    def __padding_if_neccesary(self, features_to_pad, max_size, padding_value):
        """Method for padding features if  it is required.
        
        Args:
            features_to_pad: A list of features
            max_size: Max size of the vector
            padding_value: The value of the padding elements
        
        Return:
            If the size of features_to_pad is less than max_size, a new vector
            with the elements of features_to_pad plus the elements until max_size with 
            the padding_value.
            If the size of features_to_pad is the same, the same list is returned
            If the size of features to pad is bigger that max_size, the features_to_pad
            is truncated 
        """
        len_features_to_pad = len(features_to_pad)
        if (len(features_to_pad) < max_size):
            features_to_pad += [padding_value] * (max_size - len_features_to_pad)
        else:
            features_to_pad = features_to_pad[:max_size]
        
        return(features_to_pad)
        
    
    def __doc_index_to_process(self, vector_of_ids, batch):
        """Determine the set of documents to process
        
        Args:
            vector_of_ids: The vector of doc ids
            
            batch: The batch that is being processed
            
        Return:
            A list with the ids to process
        """
        start_index = batch * self.__size_of_batches
        end_index = (batch + 1) * self.__size_of_batches
        batch_doc_ids = vector_of_ids[start_index:end_index]
        if(len(batch_doc_ids) != 0):
            if(len(batch_doc_ids) < self.__size_of_batches):
                batch_doc_ids += vector_of_ids[0:(self.__size_of_batches - len(batch_doc_ids))]
            
        else:
            batch_doc_ids = None
                
        return(batch_doc_ids)
    
    
    
        
    
    
    def __define_format_to_print(self, prefix):
        """Define the 
        """
        
        format_to_print = prefix + "Step {:d}; Epoch {:d}/{:d}, batch {:d}/{:d}: loss: {:f}; "
        for label_name in SemanticRelationsCorpusAltLex.__members__:
            format_to_print += "Precision " + label_name + ": {:f}; "
            
        for label_name in SemanticRelationsCorpusAltLex.__members__:
            format_to_print += "Recall " + label_name + ": {:f}; "
            
        for label_name in SemanticRelationsCorpusAltLex.__members__:
            format_to_print += "F1 " + label_name + ": {:f}; "
            
        format_to_print += "Precision: {:f}, Recall {:f}, F1: {:f}, Accuracy: {:f}"
        
        return(format_to_print)
    
    
    def __build_input_document_matrices(self):
        """Build the input matrices of the neural network for the text before
        and after the lexical markers
        
        Returns:
            A tuple with the four matrices matrices. The first position (0) is for the
            texts before the lexical marker, the second one (1) for the text
            after the lexical marker, the third one (2) is the list of real
            labels and the fourth one is a list with the indexes of the documents.
        """
        text_one_input_matrix = []
        text_two_input_matrix = []
        y_real_labels = []
        doc_index = []
        own_text_one_input_matrix_append = text_one_input_matrix.append
        own_text_two_input_matrix_append = text_two_input_matrix.append
        own_y_real_labels_append = y_real_labels.append
        own_doc_index_append = doc_index.append
        index = 0
        max_length_text_two = self.__max_length_lexical_marker + self.__max_length_text_two + 1
        pattern_last_token_dot = re.compile(r"\.")
        for id in self.__training_corpus.get_corpus_ids():
            doc = self.__training_corpus.get_document(id)
            text_one_features = doc.get_value_name_feature("EMB_INDEX_TEXT_ONE")
            text_one_features = self.__padding_if_neccesary(text_one_features, self.__max_length_text_one, 0)
            own_text_one_input_matrix_append(text_one_features)
            
            text_two_features = doc.get_value_name_feature("EMB_INDEX_CAUSAL_MARKER")
            text_two_features += [1] + doc.get_value_name_feature("EMB_INDEX_TEXT_TWO")
            length_before_padding = len(text_two_features)
            text_two_features = self.__padding_if_neccesary(text_two_features, max_length_text_two, 0)
            if(length_before_padding > len(text_two_features)):
                if(pattern_last_token_dot.fullmatch(doc.tokens_text_two[-1]) is not None):
                    text_two_features[-1] = self.__word_embedding_handle.get_word_index(".")
            own_text_two_input_matrix_append(text_two_features)
            
            own_y_real_labels_append(doc.real_label)
            own_doc_index_append(index)
            
            index += 1
        
        return((text_one_input_matrix, text_two_input_matrix, y_real_labels, doc_index))
    
    def __next_batch(self, batch, input_matrices_labels_ids):
        """Calculates the start and the end index of each batch and return the
        set of documents and labels of the batch.
        
        Returns:
        
        """
        
        start_index = batch * self.__size_of_batches
        end_index = (batch + 1) * self.__size_of_batches
        length_corpus = len(self.__training_corpus.corpus)
        if(end_index > length_corpus):
            end_index = length_corpus
            
        input_batch_matrix_text_one = []
        input_batch_matrix_text_two = []
        input_batch_matrix_lexical_marker = []
        input_batch_y_real_label = []
        own_input_batch_matrix_text_one = input_batch_matrix_text_one.append
        own_input_batch_matrix_text_two = input_batch_matrix_text_two.append
        own_input_batch_matrix_lexical_marker = input_batch_matrix_lexical_marker.append
        own_input_batch_y_real_label = input_batch_y_real_label.append
        for i in input_matrices_labels_ids[-1][start_index:end_index]:
            own_input_batch_matrix_text_one(input_matrices_labels_ids[0][i])
            own_input_batch_matrix_text_two(input_matrices_labels_ids[1][i])
            own_input_batch_matrix_lexical_marker(input_matrices_labels_ids[2][i])
            own_input_batch_y_real_label(input_matrices_labels_ids[3][i])
            
        return((input_batch_matrix_text_one, input_batch_matrix_text_two, input_batch_matrix_lexical_marker, input_batch_y_real_label))
    
    def training(self):
        """Train the model of the application.
        
        """
#         merged_summaries = tf.merge_all_summaries()        
        logging.info("ALTLEX_SENTENCE_ENCODING_LSTM: Beginning of the training")
        number_of_batches = int(self.__training_corpus.number_of_documents() / self.__size_of_batches)
        step = 0
        min_loss = 100000
        
        #Prepartion of the input matrices
        logging.debug("ALTLEX_SENTENCE_ENCODING_LSTM: Beggining of the building of the input matrices")
        input_matrices_labels_ids = self.__build_input_document_matrices()
        logging.debug("ALTLEX_SENTENCE_ENCODING_LSTM: Ending of the building of the input matrices")

        train_fetches = {"train_step":self.__train_step,
                                     "accuracy":self.__accuracy,
                                     "f_loss":self.__f_loss,
                                     "prediction_labels_values":self.__prediction_labels_values,
                                     "train_summary_average_loss": self.__train_summary_average_loss,
                                     "train_summary_average_accuracy": self.__train_summary_average_accuracy,
                                     "train_summary_encoding_text_one": self.__train_histogram_encoding_text_one,
                                     "train_summary_encoding_text_two": self.__train_histogram_encoding_text_two
                        }
        
        train_fetches_values = None
        
        
        #Preparation of development data if there is
        dev_text_one_matrix = None
        dev_text_two_matrix = None
        dev_y_real_labels = None
        dev_fetches = None
        dev_fetches_values = None
        
        #Preparation of DEV document
        logging.debug("ALTLEX_SENTENCE_ENCODING_LSTM: Beggining the preparation of the features of Dev dataset")
        self.__meta_corpus.build_dev_documents_features(self.__word_embedding_handle)
        dev_text_one_matrix, dev_text_two_matrix, dev_y_real_labels = self.__non_training_input_model_preparation(self.__dev_corpus)
        logging.debug("ALTLEX_SENTENCE_ENCODING_LSTM: Ending the preparation of the features of Dev dataset")
        dev_fetches = {"accuracy": self.__accuracy,
                       "f_loss": self.__f_loss,
                       "prediction_labels_values":self.__prediction_labels_values,
                       "dev_summary_accuracy": self.__dev_summary_accuracy,
                       "dev_summary_loss": self.__dev_summary_loss_function,
                       "dev_histogram_encoding_text_one":self.__dev_histogram_encoding_text_one,
                       "dev_histogram_encoding_text_two":self.__dev_histogram_encoding_text_two
                       }
        
        
        #Start training
        self.__model_session = tf.Session()
        self.__model_session.run(tf.initialize_all_variables())
        
        #Config Early stopping
        if(self.__nn_envirionment_model_dir is not None):
            self.__checkpoint_saver = tf.train.Saver(tf.all_variables())
        
        if(self.__nn_envirionment_log_dir is not None):
            self.__tf_train_writer = tf.train.SummaryWriter(self.__nn_envirionment_log_dir, self.__model_session.graph)
        
        
        losses = []
        accuracies = []
        precisions = []
        precisions_average = 0.0
        recalls = []
        recalls_average = 0.0
        f1_values = []
        f1_average = 0.0
        average_loss_per_monitor_step = 0.0
        average_accuracy_per_monitor_step = 0.0
        
        format_to_print_trainining = self.__define_format_to_print("Training: ")
        format_to_print_development = self.__define_format_to_print("Test: ")
        
        for epoch in range(self.__number_of_epochs):
            #Shuffle the index of the documents
            np.random.shuffle(input_matrices_labels_ids[-1])
            for batch in range(number_of_batches):
                input_matrices_labels = self.__next_batch(batch, input_matrices_labels_ids)
                text_one_input_matrix = input_matrices_labels[0]
                text_two_input_matrix = input_matrices_labels[1]
                y_real_labels = input_matrices_labels[2]
                
                if(self.__drop_out_training_keep_prob is not None):
                    train_fetches_values = self.__model_session.run(train_fetches, feed_dict={self.__x_text_one_placeholder:text_one_input_matrix, 
                                                                                          self.__x_text_two_placeholder:text_two_input_matrix,
                                                                                          self.__y_real_labels_placeholder:y_real_labels,
                                                                                          self.__drop_out_training_text_one_placeholder:self.__drop_out_training_keep_prob,
                                                                                          self.__drop_out_training_text_two_placeholder:self.__drop_out_training_keep_prob,
                                                                                          self.__train_summary_average_loss_function_placeholder:average_loss_per_monitor_step,
                                                                                          self.__train_summary_average_accuracy_placeholder:average_accuracy_per_monitor_step})
                else:
                    train_fetches_values = self.__model_session.run(train_fetches, feed_dict={self.__x_text_one_placeholder:text_one_input_matrix, 
                                                                                          self.__x_text_two_placeholder:text_two_input_matrix,
                                                                                          self.__y_real_labels_placeholder:y_real_labels,
                                                                                          self.__train_summary_average_loss_function_placeholder:average_loss_per_monitor_step,
                                                                                          self.__train_summary_average_accuracy_placeholder:average_accuracy_per_monitor_step})
                
                losses.append(train_fetches_values["f_loss"])
                average_loss_per_monitor_step = np.mean(losses)
                accuracies.append(train_fetches_values["accuracy"])
                average_accuracy_per_monitor_step = np.mean(accuracies)
                #This is possible because of numpy
                
                #Evaluation metrics calculation.
                precision_batch = [precision_score(y_real_labels, train_fetches_values["prediction_labels_values"], average="binary", pos_label=i) for i in range(len(SemanticRelationsCorpusAltLex.__members__))]
                precisions += [precision_batch]
                precisions_average = np.mean(precisions, 0)
                recall_batch = [recall_score(y_real_labels, train_fetches_values["prediction_labels_values"], average="binary", pos_label=i)  for i in range(len(SemanticRelationsCorpusAltLex.__members__))]
                recalls += [recall_batch]
                recalls_average = np.mean(recalls, 0)
                f1_batch = [f1_score(y_real_labels, train_fetches_values["prediction_labels_values"], average="binary", pos_label=i)  for i in range(len(SemanticRelationsCorpusAltLex.__members__))]
                f1_values += [f1_batch]
                f1_average = np.mean(f1_values, 0)
                
                #The asterisk is to expand the values of a list
                logging.debug(format_to_print_trainining.format(step-1, epoch, self.__number_of_epochs, batch, number_of_batches, average_loss_per_monitor_step, *np.concatenate((precisions_average, recalls_average, f1_average, np.array([np.mean(precisions_average)]), np.array([np.mean(recalls_average)]), np.array([np.mean(f1_average)]), np.array([average_accuracy_per_monitor_step])), axis=0)))
                step += 1
                
            #End of all the batches of the Epoch, but we are still inside the epoch loop        
            if(self.__tf_train_writer is not None):
                self.__tf_train_writer.add_summary(train_fetches_values["train_summary_average_loss"], step)
                self.__tf_train_writer.add_summary(train_fetches_values["train_summary_average_accuracy"], step)
                self.__tf_train_writer.add_summary(train_fetches_values["train_summary_encoding_text_one"], step)
                self.__tf_train_writer.add_summary(train_fetches_values["train_summary_encoding_text_two"], step)
            
            logging.info("Results EPOCH " + str(epoch))
            logging.debug(format_to_print_trainining.format(step-1, epoch, self.__number_of_epochs, batch, number_of_batches, average_loss_per_monitor_step, *np.concatenate((precisions_average, recalls_average, f1_average, np.array([np.mean(precisions_average)]), np.array([np.mean(recalls_average)]), np.array([np.mean(f1_average)]), np.array([average_accuracy_per_monitor_step])), axis=0)))
            if(self.__progress_file is not None):
                self.__progress_file.write("\n" + format_to_print_trainining.format(step-1, epoch, self.__number_of_epochs, batch, number_of_batches, average_loss_per_monitor_step, *np.concatenate((precisions_average, recalls_average, f1_average, np.array([np.mean(precisions_average)]), np.array([np.mean(recalls_average)]), np.array([np.mean(f1_average)]), np.array([average_accuracy_per_monitor_step])), axis=0))) 

            losses=[]
            accuracies=[]
            precisions = []
            recalls = []
            f1_values = []
            
            
            
            if(self.__drop_out_training_keep_prob is not None):
                dev_fetches_values = self.__model_session.run(dev_fetches, feed_dict={self.__x_text_one_placeholder:dev_text_one_matrix, 
                                                                                      self.__x_text_two_placeholder:dev_text_two_matrix, 
                                                                                      self.__y_real_labels_placeholder:dev_y_real_labels,
                                                                                      self.__drop_out_training_text_one_placeholder:1.0,
                                                                                      self.__drop_out_training_text_two_placeholder:1.0})
            else:
                dev_fetches_values = self.__model_session.run(dev_fetches, feed_dict={self.__x_text_one_placeholder:dev_text_one_matrix, 
                                                                                      self.__x_text_two_placeholder:dev_text_two_matrix, 
                                                                                      self.__y_real_labels_placeholder:dev_y_real_labels
                                                                                      })
                
            if(self.__tf_train_writer is not None):
                self.__tf_train_writer.add_summary(dev_fetches_values["dev_summary_accuracy"], step)
                self.__tf_train_writer.add_summary(dev_fetches_values["dev_summary_loss"], step)
                self.__tf_train_writer.add_summary(dev_fetches_values["dev_histogram_encoding_text_one"], step)
                self.__tf_train_writer.add_summary(dev_fetches_values["dev_histogram_encoding_text_two"], step)
            
            
            dev_precisions_batch = [precision_score(dev_y_real_labels, dev_fetches_values["prediction_labels_values"], average="binary", pos_label=i) for i in range(len(SemanticRelationsCorpusAltLex.__members__))]
            dev_recalls_batch = [recall_score(dev_y_real_labels, dev_fetches_values["prediction_labels_values"], average="binary", pos_label=i) for i in range(len(SemanticRelationsCorpusAltLex.__members__))]
            dev_f1_values_batch = [f1_score(dev_y_real_labels, dev_fetches_values["prediction_labels_values"], average="binary", pos_label=i) for i in range(len(SemanticRelationsCorpusAltLex.__members__))]
            
            logging.info(format_to_print_development.format(step-1, epoch, self.__number_of_epochs, batch, number_of_batches, dev_fetches_values["f_loss"], *np.concatenate((dev_precisions_batch, dev_recalls_batch, dev_f1_values_batch, np.array([np.mean(dev_precisions_batch)]), np.array([np.mean(dev_recalls_batch)]), np.array([np.mean(dev_f1_values_batch)]), np.array([dev_fetches_values["accuracy"]])))))
            if(self.__progress_file is not None):
                self.__progress_file.write("\n" + format_to_print_development.format(step-1, epoch, self.__number_of_epochs, batch, number_of_batches, dev_fetches_values["f_loss"], *np.concatenate((dev_precisions_batch, dev_recalls_batch, dev_f1_values_batch, np.array([np.mean(dev_precisions_batch)]), np.array([np.mean(dev_recalls_batch)]), np.array([np.mean(dev_f1_values_batch)]), np.array([dev_fetches_values["accuracy"]])))))
            
            #Store the best model
            if(dev_fetches_values["f_loss"] < min_loss):
                min_loss = dev_fetches_values["f_loss"]
                self.__best_batch = batch
                self.__best_epoch = epoch
                if(self.__checkpoint_saver is not None):
                    logging.info("SNLI_BASE_ENCODING_LSTM: Starting recording the checkpoint")
                    self.__checkpoint_saver.save(self.__model_session, self.__nn_envirionment_model_dir + "best_model_" + str(epoch) + "_" + str(batch) + "_" + str(step))
                    logging.info("SNLI_BASE_ENCODING_LSTM: Ending recording the checkpoint")
            
        logging.info("SNLI_BASE_ENCODING_LSTM: Ending of the training")
        
    def evaluation(self):
        """Evaluate the model with the test data
        """
        
        #Preparation of development data if there is
        logging.info("ALTLEX_SENTENCE_ENCODING_LSTM: Beginning of the evaluation")
        test_text_one_matrix = None
        test_text_two_matrix = None
        test_y_real_labels = None
        test_fetches_values = None
        
        self.__meta_corpus.build_test_documents_features(self.__word_embedding_handle)
        test_text_one_matrix, test_text_two_matrix, test_y_real_labels = self.__non_training_input_model_preparation(self.__test_corpus)
            
        test_fetches = {"accuracy": self.__accuracy, "f_loss": self.__f_loss, "prediction_labels_values": self.__prediction_labels_values}
        
        
        
        
        if(self.__drop_out_training_keep_prob is not None):
            test_fetches_values = self.__model_session.run(test_fetches, feed_dict={self.__x_text_one_placeholder:test_text_one_matrix, 
                                                           self.__x_text_two_placeholder:test_text_two_matrix,
                                                           self.__y_real_labels_placeholder:test_y_real_labels,
                                                           self.__drop_out_training_text_one_placeholder:1.0,
                                                           self.__drop_out_training_text_two_placeholder:1.0
                                                           })
        else:
            test_fetches_values = self.__model_session.run(test_fetches, feed_dict={self.__x_text_one_placeholder:test_text_one_matrix, 
                                                           self.__x_text_two_placeholder:test_text_two_matrix,
                                                           self.__y_real_labels_placeholder:test_y_real_labels})
            
        
        logging.info("ALTLEX_SENTENCE_ENCODING_LSTM: End of the execution of the evaluation")
        
        test_doc_ids = self.__test_corpus.get_corpus_ids()
        own_get_document = self.__test_corpus.get_document
        for test_doc_id in test_doc_ids:
            doc = own_get_document(test_doc_id)
            doc.predicted_label = test_fetches_values["prediction_labels_values"][test_doc_id]
            doc.predicted_label_best = doc.predicted_label
        
        
        self.__evaluation_precision_per_class = [precision_score(test_y_real_labels, test_fetches_values["prediction_labels_values"], average="binary", pos_label=i) for i in range(len(SemanticRelationsCorpusAltLex.__members__))]
        self.__evaluation_precision_per_class_best_model = self.__evaluation_precision_per_class
        self.__evaluation_recall_per_class = [recall_score(test_y_real_labels, test_fetches_values["prediction_labels_values"], average="binary", pos_label=i) for i in range(len(SemanticRelationsCorpusAltLex.__members__))]
        self.__evaluation_recall_per_class_best_model = self.__evaluation_recall_per_class
        self.__evaluation_f1_per_class = [f1_score(test_y_real_labels, test_fetches_values["prediction_labels_values"], average="binary", pos_label=i) for i in range(len(SemanticRelationsCorpusAltLex.__members__))]
        self.__evaluation_f1_per_class_best_model = self.__evaluation_f1_per_class
        self.__evaluation_accuracy_value = test_fetches_values["accuracy"]
        self.__evaluation_accuracy_value_best_model = self.__evaluation_accuracy_value
        self.__evaluation_loss_value = test_fetches_values["f_loss"]
        self.__evaluation_loss_value_best_model = self.__evaluation_loss_value
        self.__evaluation_precision_general = np.mean(self.__evaluation_precision_per_class)
        self.__evaluation_precision_general_best_model = self.__evaluation_precision_general
        self.__evaluation_recall_general = np.mean(self.__evaluation_recall_per_class)
        self.__evaluation_recall_general_best_model = self.__evaluation_recall_general
        self.__evaluation_f1_general = np.mean(self.__evaluation_f1_per_class)
        self.__evaluation_f1_general_best_model = self.__evaluation_f1_general
        
        if(self.__checkpoint_saver is not None):
            if(self.__checkpoint_saver.last_checkpoints is not None):
                if(len(self.__checkpoint_saver.last_checkpoints) > 0):
                    logging.info("ALTLEX_SENTENCE_ENCODING_LSTM: Starting of taking the best model")
                    self.__checkpoint_saver.restore(self.__model_session, self.__checkpoint_saver.last_checkpoints[-1])
                
                
                    if(self.__drop_out_training_keep_prob is not None):
                        test_fetches_values = self.__model_session.run(test_fetches, feed_dict={self.__x_text_one_placeholder:test_text_one_matrix, 
                                                                       self.__x_text_two_placeholder:test_text_two_matrix,
                                                                       self.__y_real_labels_placeholder:test_y_real_labels,
                                                                       self.__drop_out_training_text_one_placeholder:1.0,
                                                                       self.__drop_out_training_text_two_placeholder:1.0})
                    else:
                        test_fetches_values = self.__model_session.run(test_fetches, feed_dict={self.__x_text_one_placeholder:test_text_one_matrix, 
                                                                       self.__x_text_two_placeholder:test_text_two_matrix,
                                                                       self.__y_real_labels_placeholder:test_y_real_labels})
                    
                    
                    for test_doc_id in test_doc_ids:
                        doc = own_get_document(test_doc_id)
                        doc.predicted_label_best = test_fetches_values["prediction_labels_values"][test_doc_id]
                    
                    self.__evaluation_precision_per_class_best_model = [precision_score(test_y_real_labels, test_fetches_values["prediction_labels_values"], average="binary", pos_label=i) for i in range(len(SemanticRelationsCorpusAltLex.__members__))]
                    self.__evaluation_recall_per_class_best_model = [recall_score(test_y_real_labels, test_fetches_values["prediction_labels_values"], average="binary", pos_label=i) for i in range(len(SemanticRelationsCorpusAltLex.__members__))]
                    self.__evaluation_f1_per_class_best_model = [f1_score(test_y_real_labels, test_fetches_values["prediction_labels_values"], average="binary", pos_label=i) for i in range(len(SemanticRelationsCorpusAltLex.__members__))]
                    
                    self.__evaluation_precision_general_best_model = np.mean(self.__evaluation_precision_per_class_best_model)
                    self.__evaluation_recall_general_best_model = np.mean(self.__evaluation_recall_per_class_best_model)
                    self.__evaluation_f1_general_best_model = np.mean(self.__evaluation_f1_per_class_best_model)
                    self.__evaluation_accuracy_value_best_model = test_fetches_values["accuracy"]
                    self.__evaluation_loss_value_best_model = test_fetches_values["f_loss"]
                    logging.info("ALTLEX_SENTENCE_ENCODING_LSTM: Ending of taking the best model")
            
        logging.info("ALTLEX_SENTENCE_ENCODING_LSTM: Ending of the evaluation")
        
        
    def run_model_cross_validation(self):
        """Run the model following a cross_validation approach.
        """
        ...
        
    def closing(self):
        """Method for releasing the resources of the classification process.
        """
        self.__model_session.close()
        if(self.__progress_file is not None):
            self.__progress_file.close()
        if(self.__tf_train_writer is not None):
            self.__tf_train_writer.close()