<commit_before>import tensorflow as tf
from Models.sequenceNet import NeuralNet
from abc import abstractmethod, ABCMeta
import cPickle as Pickle
import numpy as np
import random
from PreProcessingScript.data2tensor import Mapper


class Simple(NeuralNet):
    __metaclass__ = ABCMeta

    def __init__(self, docSum_file, checkpointSys, attention=False):
        
        #A Simple (Unidirectional, One Layer) Seq2Seq Encoder-Decoder model

        #param docSum_file: the DUC Dataset in .csv format
        #param checkpointSys: The checkpoint handling object [Object]
        #param attention: True, if attention mechanism is to be implemented, else False. Default: False

        self.test_review = None
        self.predicted_test_summary = None
        self.true_summary = None
        self.train_size = None
        self.test_size = None
        self.X = None
        self.Y = None
        self.prev_mem = None
        self.cell = None
        self.dec_outputs = None
        self.dec_memory = None
        self.labels = None
        self.loss = None
        self.weights = None
        self.optimizer = None
        self.train_op = None
        self.mapper_dict = None
        self.seq_length = None
        self.vocab_size = None
        self.momentum = None

        self.attention = attention
        self.docSum_file = docSum_file
        self.checkpointSys = checkpointSys

        self.enc_inp = None
        self.dec_inp = None

        self._load_data()
        super(Simple, self).__init__()

    @abstractmethod
    def get_cell(self):
        pass

    def _split_train_tst(self):

        #divide the data into training and testing datasets 
        #Create the X_trn, X_tst, for both forward and backward, and Y_trn and Y_tst_fwd
        #Note that only the documents are changed, and not the summary.

        #:return: None

        num_samples = self.Y.shape[0]
        mapper_file = self.checkpointSys.get_mapper_file_location()
        if not self.checkpointSys.is_mapper_checkpointed():
            print 'No mapper checkpoint found. Fresh loading in progress ...'
            # Now shuffle the data
            sample_id = range(num_samples)
            random.shuffle(sample_id)
            print 'Dumping the mapper shuffle for reuse.'
            Pickle.dump(sample_id, open(mapper_file, 'wb'))
            print 'Dump complete. Moving Forward...'
        else:
            print 'Mapper Checkpoint found... Reading from mapper dump'
            sample_id = Pickle.load(open(mapper_file, 'rb'))
            print 'Mapping unpickling complete.. Moving forward...'

        self.X = self.X[sample_id]
        self.Y = self.Y[sample_id]
        # Now divide the data into test ans train set
        test_fraction = 0.15
        self.test_size = int(test_fraction * num_samples)
        self.train_size = num_samples - self.test_size
        # review
        self.X_trn = self.X[0:self.train_size]
        self.X_tst = self.X[self.train_size:num_samples]
        # Summary
        self.Y_trn = self.Y[0:self.train_size]
        self.Y_tst = self.Y[self.train_size:num_samples]

    def _load_data(self):
        """
        Load data only if the present data is not checkpointed, else, just load the checkpointed data

        :return: None
        """
        self.mapper = Mapper()
        self.mapper.generate_vocabulary(self.review_summary_file)
        self.X, self.Y = self.mapper.get_tensor()
        # Store all the mapper values in a dict for later recovery<commit_msg>Update simple.py<commit_after>import tensorflow as tf
from Models.sequenceNet import NeuralNet
from abc import abstractmethod, ABCMeta
import cPickle as Pickle
import numpy as np
import random
from PreProcessingScript.data2tensor import Mapper


class Simple(NeuralNet):
    __metaclass__ = ABCMeta

    def __init__(self, docSum_file, checkpointSys, attention=False):
        
        #A Simple (Unidirectional, One Layer) Seq2Seq Encoder-Decoder model

        #param docSum_file: the DUC Dataset in .csv format
        #param checkpointSys: The checkpoint handling object [Object]
        #param attention: True, if attention mechanism is to be implemented, else False. Default: False

        self.test_review = None
        self.predicted_test_summary = None
        self.true_summary = None
        self.train_size = None
        self.test_size = None
        self.X = None
        self.Y = None
        self.prev_mem = None
        self.cell = None
        self.dec_outputs = None
        self.dec_memory = None
        self.labels = None
        self.loss = None
        self.weights = None
        self.optimizer = None
        self.train_op = None
        self.mapper_dict = None
        self.seq_length = None
        self.vocab_size = None
        self.momentum = None

        self.attention = attention
        self.docSum_file = docSum_file
        self.checkpointSys = checkpointSys

        self.enc_inp = None
        self.dec_inp = None

        self._load_data()
        super(Simple, self).__init__()

    @abstractmethod
    def get_cell(self):
        pass

    def _split_train_tst(self):

        #divide the data into training and testing datasets 
        #Create the X_trn, X_tst, for both forward and backward, and Y_trn and Y_tst_fwd
        #Note that only the documents are changed, and not the summary.

        #:return: None

        num_samples = self.Y.shape[0]
        mapper_file = self.checkpointSys.get_mapper_file_location()
        if not self.checkpointSys.is_mapper_checkpointed():
            print 'No mapper checkpoint found. Fresh loading in progress ...'
            # Now shuffle the data
            sample_id = range(num_samples)
            random.shuffle(sample_id)
            print 'Dumping the mapper shuffle for reuse.'
            Pickle.dump(sample_id, open(mapper_file, 'wb'))
            print 'Dump complete. Moving Forward...'
        else:
            print 'Mapper Checkpoint found... Reading from mapper dump'
            sample_id = Pickle.load(open(mapper_file, 'rb'))
            print 'Mapping unpickling complete.. Moving forward...'

        self.X = self.X[sample_id]
        self.Y = self.Y[sample_id]
        # Now divide the data into test ans train set
        test_fraction = 0.15
        self.test_size = int(test_fraction * num_samples)
        self.train_size = num_samples - self.test_size
        # review
        self.X_trn = self.X[0:self.train_size]
        self.X_tst = self.X[self.train_size:num_samples]
        # Summary
        self.Y_trn = self.Y[0:self.train_size]
        self.Y_tst = self.Y[self.train_size:num_samples]

    def _load_data(self):
        """
        Load data only if the present data is not checkpointed, else, just load the checkpointed data

        :return: None
        """
        self.mapper = Mapper()
        self.mapper.generate_vocabulary(self.docSum_file)
        self.X, self.Y = self.mapper.get_tensor()
        # Store all the mapper values in a dict for later recovery
{'fraction_non_alphanumeric': 0.07527496071989716, 'fraction_numerical': 0.0022853878017426083, 'mean_word_length': 2.2431681333950904, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '180671', 'n_tokens_mistral': 2093, 'n_tokens_neox': 2010, 'n_words': 580}