<commit_before># strconv.py
# Copyright (c) 2013 <NAME>
# BSD License

__version__ = '0.4.0'

from collections import Counter


class TypeInfo(object):
    "Sampling and frequency of a type for a sample of values."
    def __init__(self, name, size=None, total=None):
        self.name = name
        self.count = 0
        self.sample = []
        self.size = size
        self.total = total
        self.sample_set = set()

    def __repr__(self):
        return '<{0}: {1} n={2}>'.format(self.__class__.__name__,
                                         self.name, self.count)

    def incr(self, n=1):
        self.count += n

    def add(self, i, value):
        if self.size is None or len(self.sample) < self.size:
            # No dupes
            if value not in self.sample_set:
                self.sample_set.add(value)
                self.sample.append((i, value))

    def freq(self):<commit_msg>0.4.1 Release<commit_after># strconv.py
# Copyright (c) 2013 <NAME>
# BSD License

__version__ = '0.4.1'

from collections import Counter


class TypeInfo(object):
    "Sampling and frequency of a type for a sample of values."
    def __init__(self, name, size=None, total=None):
        self.name = name
        self.count = 0
        self.sample = []
        self.size = size
        self.total = total
        self.sample_set = set()

    def __repr__(self):
        return '<{0}: {1} n={2}>'.format(self.__class__.__name__,
                                         self.name, self.count)

    def incr(self, n=1):
        self.count += n

    def add(self, i, value):
        if self.size is None or len(self.sample) < self.size:
            # No dupes
            if value not in self.sample_set:
                self.sample_set.add(value)
                self.sample.append((i, value))

    def freq(self):
{'fraction_non_alphanumeric': 0.11842105263157894, 'fraction_numerical': 0.014802631578947368, 'mean_word_length': 1.869496855345912, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6216884', 'n_tokens_mistral': 582, 'n_tokens_neox': 555, 'n_words': 158}