<commit_before># Copyright (C) 2010 <NAME>

import math

class Statistics:
    def __init__(self):
        self.statistics = {}

    def add_sample(self, stat_name, sample):
        stat = self.statistics.setdefault(stat_name, Histogram(stat_name))
        stat.add_sample(sample)

class Histogram:
    def __init__(self, name):
        self._name = name
        self._samples = []

        self._buckets = {}

    def _get_bucket(self, sample):
        if sample == 0:
            bucket = 0
        else:
            # use exponential buckets
            bucket = math.floor(math.exp(math.floor(math.log(sample))))

        return self._buckets.setdefault(bucket, [])

    def add_sample(self, sample):
        bucket = self._get_bucket(sample)
        bucket.append(sample)

    def stats(self):
        buckets = self._buckets.keys()
        buckets.sort()

        count = 0
        total = 0

        stats = {}

        for bucket_id in buckets:
            samples = self._buckets[bucket_id]

            bucket_stats = self._bucket_stats(samples)

            stats.setdefault("buckets", []).append((bucket_id, bucket_stats))

            count = count + bucket_stats["count"]
            total = total + bucket_stats["total"]

        stats["count"] = count
        stats["total"] = total

        stats["mean"] = mean = float(total)/count
        sum_squares = 0

        # calculate the standard deviation of the whole set
        for bucket in self._buckets.values():
            for sample in bucket:
                sum_squares = sum_squares + pow(sample-mean, 2)

        stats["stddev"] = math.sqrt(sum_squares/count)

        return stats

    def _bucket_stats(self, samples):
        count = len(samples)
        total = 0
        sum_squares = 0

        for sample in samples:
            total = total + sample

        mean = float(total) / count

        sum_squares = 0
        for sample in samples:
            sum_squares = sum_squares + pow(sample-mean, 2)

        stddev = math.sqrt(sum_squares / count)

        stats = { "count" : count,
                  "total" : total,
                  "mean" : mean,
                  "stddev" : stddev }

        return stats

    def text(self, fd):
        bar_width = 60
        stats = self.stats()

        fd.write("\n")
        fd.write("Histogram: %s recorded %d samples, avg = %.1f, std = %.1f\n"
                 % (self._name, stats["count"], stats["mean"], stats["stddev"]))

        # largest sample count of all buckets
        max_count = 0
        for bucket, bucket_stats in stats["buckets"]:
            if bucket_stats["count"] > max_count:
                max_count = bucket_stats["count"]

        for bucket, bucket_stats in stats["buckets"]:
            pct = float(bucket_stats["count"]) / stats["count"] * 100

            graph = '=' * int(bar_width * pct/100) + "O"

            bar = graph + (' ' * (bar_width-len(graph)))

            fd.write("%-6d %s (%.1f%%)\n" % (bucket, bar, pct))<commit_msg>Add sample counts to each bucket's output in histograms<commit_after># Copyright (C) 2010 <NAME>

import math

class Statistics:
    def __init__(self):
        self.statistics = {}

    def add_sample(self, stat_name, sample):
        stat = self.statistics.setdefault(stat_name, Histogram(stat_name))
        stat.add_sample(sample)

class Histogram:
    def __init__(self, name):
        self._name = name
        self._samples = []

        self._buckets = {}

    def _get_bucket(self, sample):
        if sample == 0:
            bucket = 0
        else:
            # use exponential buckets
            bucket = math.floor(math.exp(math.floor(math.log(sample))))

        return self._buckets.setdefault(bucket, [])

    def add_sample(self, sample):
        bucket = self._get_bucket(sample)
        bucket.append(sample)

    def stats(self):
        buckets = self._buckets.keys()
        buckets.sort()

        count = 0
        total = 0

        stats = {}

        for bucket_id in buckets:
            samples = self._buckets[bucket_id]

            bucket_stats = self._bucket_stats(samples)

            stats.setdefault("buckets", []).append((bucket_id, bucket_stats))

            count = count + bucket_stats["count"]
            total = total + bucket_stats["total"]

        stats["count"] = count
        stats["total"] = total

        stats["mean"] = mean = float(total)/count
        sum_squares = 0

        # calculate the standard deviation of the whole set
        for bucket in self._buckets.values():
            for sample in bucket:
                sum_squares = sum_squares + pow(sample-mean, 2)

        stats["stddev"] = math.sqrt(sum_squares/count)

        return stats

    def _bucket_stats(self, samples):
        count = len(samples)
        total = 0
        sum_squares = 0

        for sample in samples:
            total = total + sample

        mean = float(total) / count

        sum_squares = 0
        for sample in samples:
            sum_squares = sum_squares + pow(sample-mean, 2)

        stddev = math.sqrt(sum_squares / count)

        stats = { "count" : count,
                  "total" : total,
                  "mean" : mean,
                  "stddev" : stddev }

        return stats

    def text(self, fd):
        bar_width = 60
        stats = self.stats()

        fd.write("\n")
        fd.write("Histogram: %s recorded %d samples, avg = %.1f, std = %.1f\n"
                 % (self._name, stats["count"], stats["mean"], stats["stddev"]))

        # largest sample count of all buckets
        max_count = 0
        for bucket, bucket_stats in stats["buckets"]:
            if bucket_stats["count"] > max_count:
                max_count = bucket_stats["count"]

        for bucket, bucket_stats in stats["buckets"]:
            pct = float(bucket_stats["count"]) / stats["count"] * 100

            graph = '=' * int(bar_width * pct/100) + "O"

            bar = graph + (' ' * (bar_width-len(graph)))

            fd.write("%-6d %s (%d = %.1f%%)\n" % (bucket, bar, bucket_stats["count"], pct))
{'fraction_non_alphanumeric': 0.12022397891963109, 'fraction_numerical': 0.008893280632411068, 'mean_word_length': 1.8974236641221374, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '467439', 'n_tokens_mistral': 1897, 'n_tokens_neox': 1733, 'n_words': 439}