<commit_before># Author: <NAME>
# (c) 2016
# San Jose State University

import numpy as np
import random
import time
import sys

class insert_file_info :
    
    def __init__(self, full_file_path, filenumber, batch_size = 50, 
        use_random_seed = False ) :
        """ full_file_path : full file path of the shuffled data
            filenumber     : An array of file number """
        self.filename         = full_file_path.rsplit('\\', 1)[-1]
        self.filename         = self.filename.rsplit('/', 1)[-1]
        self.full_file_path   = full_file_path
        self.nrows            = 0
        self.ncols            = 0
        self.nfile            = len(filenumber)
        self.batch_size       = batch_size
        self.current_index    = 0

    class DataSet(object) :
        file_info = None
    
        def __init__(self, images, labels, nrows, nfile_train, 
                     nfile_test, nfile_val, full_file_path, data_type) :
            #self.file_into = insert_file_info()
        
            #super(DataSet,self).__init__()
            #self.insert_file_info = insert_file_info

            self._epochs_completed = 0
            self._file_index = 1
            self._images = images
            self._index_in_datafile = 0
            self._index_in_epoch = 0
            self._labels = labels
            self._ndata = 0
            self.batch_size = 0
            self.data_type = data_type
            self.full_file_path = full_file_path
            self.nrows = nrows
             
            if self.data_type == 'train' :
                self.start_file_index   = 1
                self.end_file_index     = nfile_train
                self._ndata             = nfile_train*self.nrows
                self.convert_to_one_hot = True
            elif self.data_type == 'test' :
                self.start_file_index   = nfile_train + 1
                self.end_file_index     = nfile_train + nfile_test
                self._ndata             = nfile_test*self.nrows
                self.convert_to_one_hot = True
            elif self.data_type == 'validation' :
                self.start_file_index   = nfile_train + nfile_test + 1
                self.end_file_index     = nfile_train + nfile_test + nfile_val
                self._ndata             = nfile_val*self.nrows
                self.convert_to_one_hot = False

        #@staticmethod
        #def feed_self(self, batch_size, nrows) :
        #    self.batch_size = batch_size
        #    self.nrows      = nrows
            #print self.batch_size, self.nrows

        @property
        def images(self):
            return self._images
        
        @property
        def labels(self):
            return self._labels
    
        @property
        def ndata(self):
            return self._ndata
    
        @property
        def epochs_completed(self):
            return self._epochs_completed
            
        def next_batch(self, batch_size = 50) :
            
            start = self._index_in_epoch
            if ( self._epochs_completed == 0 ) and ( start == 0 ) :
                self.batch_size = batch_size
                while np.modf(float(self.nrows)/self.batch_size)[0] > 0.0 :
                     print 'Warning! Number of data/ batch size must be an integer.'
                     print 'number of data: %d' % self._ndata
                     print 'batch size: %d'     % self.batch_size
                     self.batch_size = int(input('Input new batch size: '))
                print 'batch size : %d'    % self.batch_size
                print 'number of data: %d' % self._ndata

            self._index_in_epoch += self.batch_size
            if self._index_in_epoch > self._ndata :
                # Number of training epochs completed
                self._epochs_completed += 1<commit_msg>Fixed small bug.<commit_after># Author: <NAME>
# (c) 2016
# San Jose State University

import numpy as np
import random
import time
import sys

class insert_file_info :
    
    def __init__(self, full_file_path, filenumber, batch_size = 50, 
        use_random_seed = False ) :
        """ full_file_path : full file path of the shuffled data
            filenumber     : An array of file number """
        self.filename         = full_file_path.rsplit('\\', 1)[-1]
        self.filename         = self.filename.rsplit('/', 1)[-1]
        self.full_file_path   = full_file_path
        self.nrows            = 0
        self.ncols            = 0
        self.nfile            = len(filenumber)
        self.batch_size       = batch_size
        self.current_index    = 0

    class DataSet(object) :
        file_info = None
    
        def __init__(self, images, labels, nrows, nfile_train, 
                     nfile_test, nfile_val, full_file_path, data_type) :
            #self.file_into = insert_file_info()
        
            #super(DataSet,self).__init__()
            #self.insert_file_info = insert_file_info

            self._epochs_completed = 0
            self._file_index = 1
            self._images = images
            self._index_in_datafile = 0
            self._index_in_epoch = 0
            self._labels = labels
            self._ndata = 0
            self.batch_size = 0
            self.data_type = data_type
            self.full_file_path = full_file_path
            self.nrows = nrows
             
            if self.data_type == 'train' :
                self.start_file_index   = 1
                self.end_file_index     = nfile_train
                self._ndata             = nfile_train*self.nrows
                self.convert_to_one_hot = True
            elif self.data_type == 'test' :
                self.start_file_index   = nfile_train + 1
                self.end_file_index     = nfile_train + nfile_test
                self._ndata             = nfile_test*self.nrows
                self.convert_to_one_hot = True
            elif self.data_type == 'validation' :
                self.start_file_index   = nfile_train + nfile_test + 1
                self.end_file_index     = nfile_train + nfile_test + nfile_val
                self._ndata             = nfile_val*self.nrows
                self.convert_to_one_hot = False

        #@staticmethod
        #def feed_self(self, batch_size, nrows) :
        #    self.batch_size = batch_size
        #    self.nrows      = nrows
            #print self.batch_size, self.nrows

        @property
        def images(self):
            return self._images
        
        @property
        def labels(self):
            return self._labels
    
        @property
        def ndata(self):
            return self._ndata
    
        @property
        def epochs_completed(self):
            return self._epochs_completed
            
        def next_batch(self, batch_size = 50) :
            
            start = self._index_in_epoch
            if ( self._epochs_completed == 0 ) and ( start == 0 ) :
                self.batch_size = batch_size
                while np.modf(float(self._ndata)/self.batch_size)[0] > 0.0 :
                     print 'Warning! Number of data/ batch size must be an integer.'
                     print 'number of data: %d' % self._ndata
                     print 'batch size: %d'     % self.batch_size
                     self.batch_size = int(input('Input new batch size: '))
                print 'batch size : %d'    % self.batch_size
                print 'number of data: %d' % self._ndata

            self._index_in_epoch += self.batch_size
            if self._index_in_epoch > self._ndata :
                # Number of training epochs completed
                self._epochs_completed += 1
{'fraction_non_alphanumeric': 0.0770735524256651, 'fraction_numerical': 0.00782472613458529, 'mean_word_length': 1.4675032175032174, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1918106', 'n_tokens_mistral': 2273, 'n_tokens_neox': 2164, 'n_words': 481}