<commit_before>import redis


class HyperLogLogJaccardIndex(object):

    def __init__(self, host='localhost', port='6379', db=0, prefix="hll_"):
        self.storage = redis.StrictRedis(host=host, port=port, db=int(db))
        self.prefix = prefix

    def insert(self, kmers, sample):
        self.storage.pfadd(self._storage_key(sample), *kmers)

    def union(self, sample1, sample2):
        samples = [self._storage_key(sample1), self._storage_key(sample2)]
        self.storage.pfcount(*samples)
        return self.storage.pfcount(*samples)<commit_msg>Run pfadd command in chunks<commit_after>import redis


def chunks(l, n):
    """Yield successive n-sized chunks from l."""
    for i in range(0, len(l), n):
        yield l[i:i + n]


class HyperLogLogJaccardIndex(object):

    def __init__(self, host='localhost', port='6379', db=0, prefix="hll_"):
        self.storage = redis.StrictRedis(host=host, port=port, db=int(db))
        self.prefix = prefix
        self.max_N_pfadd = 100000

    def insert(self, kmers, sample):
        return self._massive_insert(kmers, sample)

    def _massive_insert(self, kmers, sample):
        for _kmers in chunks(kmers, self.max_N_pfadd):
            self.storage.pfadd(self._storage_key(sample), *_kmers)

    def union(self, sample1, sample2):
        samples = [self._storage_key(sample1), self._storage_key(sample2)]
        self.storage.pfcount(*samples)
        return self.storage.pfcount(*samples)
{'fraction_non_alphanumeric': 0.13109128345916266, 'fraction_numerical': 0.017158544955387784, 'mean_word_length': 3.5993690851735014, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4402331', 'n_tokens_mistral': 499, 'n_tokens_neox': 473, 'n_words': 99}