<commit_before>from __future__ import print_function
import os
import sys
import time
import pickle
import numpy as np



# implementing Data loader, that manages loading of data, shuffling, mini batches and selection of next mini batch from specific
class DataLoader(object):
    """ load data from a list of files """
    def __init__(self,list_of_files,file_format,minibatch_size=None):
        """
            Create a input data layer that is 2D, with features lying in rows.
            
            :type list_of_files: tuple, containing the list of files to load from

            :type input: file_format: either mat or pickle
            : in case of matlab file, the last element in the list should be pickle
            : file with obj2label dictionary. Each mat file has these keys:
            : "data":x, "joints":p, "labels":y,
            : "track":t, ["mean":mean]
            : in case of pickle files, each file is
            : x,p,y,t,obj2label
            
        """
        self.load_data(list_of_files,file_format)#load data
        self.minibatch_idx = 0#initialize mini batch index
        if minibatch_size!=None:
            self.minibatch_size = minibatch_size
        
        
    def load_data(self,list_of_files,file_format):
        if file_format=="mat":
            print( "not implemented!")
            assert(False)
        elif file_format=="pkl":
            print( "loading pickled data")
            if len(list_of_files)>1:
                xs,ps,ys,ts = [],[],[],[]
                for fname in list_of_files:
                    f = file(fname, "rb")
                    x,p,y,t,self.obj2label = pickle.load(f)
                    f.close()
                    xs.append(x)
                    ys.append(y)
                    ts.append(t)
                    ps.append(p)
                self.x = np.concantenate(xs)
                self.y = np.concantenate(ys)
                self.t = np.concantenate(ts)
                self.p = np.concantenate(ps)
            else:
                f = file(list_of_files[0], "rb")
                self.x,self.p,self.y,self.t,self.obj2label = pickle.load(f)
                f.close()
            print( "data size:",self.x.shape)
            print( "target shape:",self.y.shape)
            print( "number of target classes:",np.unique(self.y).shape)
            print( "number of tracks:",np.unique(self.t).shape[0])
            assert(self.y.shape[0]==self.x.shape[0])
            assert(self.y.shape[0]==self.t.shape[0])
            assert(self.y.shape[0]==self.p.shape[0])
        else:
            print("wrong file format!")
            assert(False)
        self.update_tracks_record()#keep track of where different tracks are
<commit_msg>pickle<commit_after>from __future__ import print_function
import os
import sys
import time
import pickle
import numpy as np



# implementing Data loader, that manages loading of data, shuffling, mini batches and selection of next mini batch from specific
class DataLoader(object):
    """ load data from a list of files """
    def __init__(self,list_of_files,file_format,minibatch_size=None):
        """
            Create a input data layer that is 2D, with features lying in rows.
            
            :type list_of_files: tuple, containing the list of files to load from

            :type input: file_format: either mat or pickle
            : in case of matlab file, the last element in the list should be pickle
            : file with obj2label dictionary. Each mat file has these keys:
            : "data":x, "joints":p, "labels":y,
            : "track":t, ["mean":mean]
            : in case of pickle files, each file is
            : x,p,y,t,obj2label
            
        """
        self.load_data(list_of_files,file_format)#load data
        self.minibatch_idx = 0#initialize mini batch index
        if minibatch_size!=None:
            self.minibatch_size = minibatch_size
        
        
    def load_data(self,list_of_files,file_format):
        if file_format=="mat":
            print( "not implemented!")
            assert(False)
        elif file_format=="pkl":
            print( "loading pickled data")
            if len(list_of_files)>1:
                xs,ps,ys,ts = [],[],[],[]
                for fname in list_of_files:
                    f = file(fname, "rb")
                    x,p,y,t,self.obj2label = pickle.load(f)
                    f.close()
                    xs.append(x)
                    ys.append(y)
                    ts.append(t)
                    ps.append(p)
                self.x = np.concantenate(xs)
                self.y = np.concantenate(ys)
                self.t = np.concantenate(ts)
                self.p = np.concantenate(ps)
            else:
                f = open(list_of_files[0], "rb")
                self.x,self.p,self.y,self.t,self.obj2label = pickle.load(f)
                f.close()
            print( "data size:",self.x.shape)
            print( "target shape:",self.y.shape)
            print( "number of target classes:",np.unique(self.y).shape)
            print( "number of tracks:",np.unique(self.t).shape[0])
            assert(self.y.shape[0]==self.x.shape[0])
            assert(self.y.shape[0]==self.t.shape[0])
            assert(self.y.shape[0]==self.p.shape[0])
        else:
            print("wrong file format!")
            assert(False)
        self.update_tracks_record()#keep track of where different tracks are

{'fraction_non_alphanumeric': 0.10775151181968114, 'fraction_numerical': 0.005497526113249038, 'mean_word_length': 1.7861153649821337, 'pattern_counts': {'":': 14, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1659592', 'n_tokens_mistral': 1642, 'n_tokens_neox': 1593, 'n_words': 404}