<commit_before>#
# A Class that couples samples from a simulation 
# with some statistics functions
#
# <NAME> (<EMAIL>)
#
# Improved by <NAME> (<EMAIL>)
from mpmath import *
import random

#
# A Class that incapsulates a set of samples with 
# operations over those samples (i.e. statistics)
#
class Samples:

    #
    # Construct new instance with a list of samples
    #
    # @param samples: a set of samples, most observed in simulation
    #
    def __init__(self):

        self.byte_sum = mpf(0)
        self.prob_sum = mpf(0)

        self.num_samples = 0

        # 
        # A static table used to estimate the confidence 
        # interval around a sample mean
        #
        self.conf_lvl_lku = {}
        self.conf_lvl_lku["0.80"] = mpf(1.281)
        self.conf_lvl_lku["0.85"] = mpf(1.440)
        self.conf_lvl_lku["0.90"] = mpf(1.645)
        self.conf_lvl_lku["0.95"] = mpf(1.960)
        self.conf_lvl_lku["0.995"] = mpf(2.801)

        self.byte_mean = None
        self.byte_dev = None
        self.byte_ci = None
        self.byte_re = None

        # used to calculate the probability of data loss
        self.prob_mean = None
        self.prob_dev = None
        self.prob_ci = None
        self.prob_re = None
        
    #
    # Calculate the sample mean based on the samples for this instance
    #
    def calcMean(self):
        self.byte_mean = self.byte_sum / self.num_samples
        self.prob_mean = self.prob_sum / self.num_samples

    #
    # Calculate the standard deviation based on the samples for this instance
    #
    def calcStdDev(self, samples):
        if self.byte_dev == None:
            self.calcMean()
            sum_1 = (self.num_samples - len(samples)) * abs(power(0 - self.byte_mean, 2))
            for sample in samples:
                sum_1 += abs(power(sample - self.byte_mean, 2))

            sum_2 = (self.num_samples - len(samples)) * abs(power(0 - self.prob_mean, 2)) + len(samples) * abs(power(1 - self.prob_mean, 2))

    
            self.byte_dev = sqrt((mpf(1)/(self.num_samples-1)) * sum_1)
            self.prob_dev = sqrt((mpf(1)/(self.num_samples-1)) * sum_2)


    #
    # Calculate the confidence interval around the sample mean 
    #
    # @param conf_level: the probability that the mean falls within the interval
    #
    def calcConfInterval(self, conf_level, samples):
        
        if conf_level not in self.conf_lvl_lku.keys():
            print "%s not a valid confidence level!" % conf_level
            return None
    
        self.calcStdDev(samples)

        self.byte_ci = abs(self.conf_lvl_lku[conf_level] * (self.byte_dev / sqrt(self.num_samples)))
        self.prob_ci = abs(self.conf_lvl_lku[conf_level] * (self.prob_dev / sqrt(self.num_samples)))

    #
    # Calculate the relative error 
    #
    # self.conf_lvl_lku[conf_level] * sqrt(Var)/sqrt(num_samples) / mean
    #
    def calcRE(self, conf_level, samples):
        if self.byte_mean == 0:
            return None
        
        self.calcConfInterval(conf_level, samples)

        self.byte_re = self.byte_ci / self.byte_mean
        self.prob_re = self.prob_ci / self.prob_mean

    # zeros have been eliminated
    def calcResults(self, conf_level, samples, sample_num):
        self.num_samples = sample_num
        self.prob_sum += len(samples)
        for sample in samples:
            self.byte_sum += sample

        self.calcRE(conf_level, samples)
    
#
# Generate samples from a known distribution and verify the statistics
#
def test():
    
    num_samples = 1000
    samples = []
    mean = 0.5
    std_dev = 0.001
    for i in range(num_samples):
        samples.append(random.gauss(mean, std_dev))

    s = Samples(1000)    
    s.calcResults("0.9", samples)
        
    print "Mean: %s (%s): " % (s.calcMean(), mean)
    print "Std Dev: %s (%s): " % (s.calcStdDev(), std_dev)
    print "Conf. Interval: (%s, %s)" % s.calcConfInterval("0.995")

    (a,b,c,d) = s.getResults()
    print "Mean: %s (%s): " % (a, mean)
    print "Conf. Interval: (%s, %s)" % (b,c)
    print "Relative Error: (%s)" % d

if __name__ == "__main__":
    test()    



<commit_msg>fix ZeroDivisionError<commit_after>#
# A Class that couples samples from a simulation 
# with some statistics functions
#
# <NAME> (<EMAIL>)
#
# Improved by <NAME> (<EMAIL>)
from mpmath import *
import random

#
# A Class that incapsulates a set of samples with 
# operations over those samples (i.e. statistics)
#
class Samples:

    #
    # Construct new instance with a list of samples
    #
    # @param samples: a set of samples, most observed in simulation
    #
    def __init__(self):

        self.byte_sum = mpf(0)
        self.prob_sum = mpf(0)

        self.num_samples = 0

        # 
        # A static table used to estimate the confidence 
        # interval around a sample mean
        #
        self.conf_lvl_lku = {}
        self.conf_lvl_lku["0.80"] = mpf(1.281)
        self.conf_lvl_lku["0.85"] = mpf(1.440)
        self.conf_lvl_lku["0.90"] = mpf(1.645)
        self.conf_lvl_lku["0.95"] = mpf(1.960)
        self.conf_lvl_lku["0.995"] = mpf(2.801)

        self.byte_mean = None
        self.byte_dev = None
        self.byte_ci = None
        self.byte_re = None

        # used to calculate the probability of data loss
        self.prob_mean = None
        self.prob_dev = None
        self.prob_ci = None
        self.prob_re = None
        
    #
    # Calculate the sample mean based on the samples for this instance
    #
    def calcMean(self):
        self.byte_mean = self.byte_sum / self.num_samples
        self.prob_mean = self.prob_sum / self.num_samples

    #
    # Calculate the standard deviation based on the samples for this instance
    #
    def calcStdDev(self, samples):
        if self.byte_dev == None:
            self.calcMean()
            sum_1 = (self.num_samples - len(samples)) * abs(power(0 - self.byte_mean, 2))
            for sample in samples:
                sum_1 += abs(power(sample - self.byte_mean, 2))

            sum_2 = (self.num_samples - len(samples)) * abs(power(0 - self.prob_mean, 2)) + len(samples) * abs(power(1 - self.prob_mean, 2))

    
            self.byte_dev = sqrt((mpf(1)/(self.num_samples-1)) * sum_1)
            self.prob_dev = sqrt((mpf(1)/(self.num_samples-1)) * sum_2)


    #
    # Calculate the confidence interval around the sample mean 
    #
    # @param conf_level: the probability that the mean falls within the interval
    #
    def calcConfInterval(self, conf_level, samples):
        
        if conf_level not in self.conf_lvl_lku.keys():
            print "%s not a valid confidence level!" % conf_level
            return None
    
        self.calcStdDev(samples)

        self.byte_ci = abs(self.conf_lvl_lku[conf_level] * (self.byte_dev / sqrt(self.num_samples)))
        self.prob_ci = abs(self.conf_lvl_lku[conf_level] * (self.prob_dev / sqrt(self.num_samples)))

    #
    # Calculate the relative error 
    #
    # self.conf_lvl_lku[conf_level] * sqrt(Var)/sqrt(num_samples) / mean
    #
    def calcRE(self, conf_level, samples):
        
        self.calcConfInterval(conf_level, samples)

        if self.byte_mean == 0:
            self.byte_re = 0
            self.prob_re = 0
        else:
            self.byte_re = self.byte_ci / self.byte_mean
            self.prob_re = self.prob_ci / self.prob_mean

    # zeros have been eliminated
    def calcResults(self, conf_level, samples, sample_num):
        self.num_samples = sample_num
        self.prob_sum += len(samples)
        for sample in samples:
            self.byte_sum += sample

        self.calcRE(conf_level, samples)
    
#
# Generate samples from a known distribution and verify the statistics
#
def test():
    
    num_samples = 1000
    samples = []
    mean = 0.5
    std_dev = 0.001
    for i in range(num_samples):
        samples.append(random.gauss(mean, std_dev))

    s = Samples(1000)    
    s.calcResults("0.9", samples)
        
    print "Mean: %s (%s): " % (s.calcMean(), mean)
    print "Std Dev: %s (%s): " % (s.calcStdDev(), std_dev)
    print "Conf. Interval: (%s, %s)" % s.calcConfInterval("0.995")

    (a,b,c,d) = s.getResults()
    print "Mean: %s (%s): " % (a, mean)
    print "Conf. Interval: (%s, %s)" % (b,c)
    print "Relative Error: (%s)" % d

if __name__ == "__main__":
    test()    




{'fraction_non_alphanumeric': 0.11551352645439311, 'fraction_numerical': 0.01843428297821403, 'mean_word_length': 2.249708284714119, 'pattern_counts': {'":': 2, '<': 11, '<?xml version=': 0, '>': 11, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2586494', 'n_tokens_mistral': 2901, 'n_tokens_neox': 2735, 'n_words': 694}