<commit_before>#!/usr/bin/env python

import csv
import scipy as sp

class DataSet:
    def __init__ (self, count = [], time = []):
        # Defines counts and times lists, while ensuring that the data being passed in is actually
        # a number (based on the data read using csv.reader(), these lists are actually strings containing
        # numbers, but this format of the data works fine with matplotlib).
        self.counts = [value for value in count if value.isdigit() == True]
        self.times = [value for value in time if value.isdigit() == True]
        
        # maxTimeResolution defines the binWidth (the time between each sampling of the data).
        # This is a number
        self.maxTimeResolution = abs(float(self.times[len(self.times)-1]) - float(self.times[len(self.times)-2]))
        
    
def readInput(filename):
    with open(filename, 'rb') as csvfile:
        # csv.reader returns a list containing strings, from a tab seperated text file.
        # Based on data taken with PRA 5, the first element in entry of the list it the time,
        # while the second is the rows
        reader = csv.reader(csvfile, delimiter='\t')
        count = []
        time = []
        for row in reader:
            time.append(row[0])
            count.append(row[1])
    
    data = DataSet(count, time)
    
    print data.counts
    print data.times
    print data.maxTimeResolution
    return data

def rebin( a, newshape):
    '''Rebin an array to a new shape.
       newshape must be a factor of a.shape.
       Source: http://www.scipy.org/Cookbook/Rebinning
    '''

    assert len(a.shape) == len(newshape)
    assert not sp.sometrue(sp.mod( a.shape, newshape ))

    slices = [ slice(None,None, old/new) for old,new in zip(a.shape,newshape) ]
    return a[slices]<commit_msg>rewrote rebin() function inside DataSet. However, it has not been tested<commit_after>#!/usr/bin/env python

import csv

# All times must be in seconds
class DataSet:
    def __init__ (self, count = [], time = []):
        # Defines lists of floats for counts and times
        self.counts = count
        self.times = times
        
        # maxTimeResolution defines the binWidth (the time between each sampling of the data).
        # This is a number
        self.maxTimeResolution = abs(self.times[len(self.times)-1] - self.times[len(self.times)-2])
    
    def rebin(self, newBinWidth):
        # Determine the factor by which to scale the bins
        rebinningFactor = newBinWidth/self.maxTimeResolution
        
        newCounts = []
        newTimes = []
        for i in range(0,len(self.times)/rebinningFactor):
            newCounts.append( sum( self.counts[ newBinWidth*i : newBinWidth*(i+1) ] ))
            newTimes.append(newBinWidth*i)
        
        newDataSet = DataSet(newCounts, newTimes)
        
        return newDataSet
    
def readInput(filename):
    with open(filename, 'rb') as csvfile:
        # csv.reader returns a list containing strings, from a tab seperated text file.
        # Based on data taken with PRA 5, the first element in entry of the list it the time,
        # while the second is the rows
        reader = csv.reader(csvfile, delimiter='\t')
        count = []
        time = []
        for row in reader:
            time.append(row[0])
            count.append(row[1])
    
    # Ensures that the data being passed in is actually a number (based on the data read using
    # csv.reader(), these lists are actually strings containing numbers, so this format must be
    # converted into numbers using float()).
    count = [float(value) for value in count if value.isdigit() == True]
    time = [float(value) for value in time if value.isdigit() == True]
    
    data = DataSet(count, time)
    
    print data.counts
    print data.times
    print data.maxTimeResolution
    return data
{'fraction_non_alphanumeric': 0.0934094447327452, 'fraction_numerical': 0.0031136481577581734, 'mean_word_length': 2.451208594449418, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 1, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2917465', 'n_tokens_mistral': 1131, 'n_tokens_neox': 1111, 'n_words': 401}