<commit_before># Copyright (C) 2012 <NAME>

import heapq
import io
import itertools
import logging
import operator
import tempfile

from cobe import model

logger = logging.getLogger("cobe.counter")


class MergeCounter(object):
    """Count unique string elements with tunable memory usage.

    MergeCounter automatically scales to lists of items that cannot
    fit into memory, writing counts to disk and yielding its results
    by merging the on-disk files.
    """
    def __init__(self, max_fds=32, max_len=4000000):
        """Init MergeCounter with tunable resource parameters.

        Args:
            max_fds: the maximum number of merge files to keep open
            max_len: the total length of strings stored before
                flushing counts to disk. This is an estimate of memory
                usage and not an accurate limit, as it doesn't include
                item counts or dict overhead.
        """

        self.max_fds = max_fds
        self.max_len = max_len

    def count(self, items):
        """Count string items.

        This method keeps an in-memory count of items until its size
        reaches max_len, then flushes the counts to a sorted overflow
        file on disk. It does this as many times as necessary, then<commit_msg>Remove unused imports found by pyflakes<commit_after># Copyright (C) 2012 <NAME>

import heapq
import logging
import operator
import tempfile

logger = logging.getLogger("cobe.counter")


class MergeCounter(object):
    """Count unique string elements with tunable memory usage.

    MergeCounter automatically scales to lists of items that cannot
    fit into memory, writing counts to disk and yielding its results
    by merging the on-disk files.
    """
    def __init__(self, max_fds=32, max_len=4000000):
        """Init MergeCounter with tunable resource parameters.

        Args:
            max_fds: the maximum number of merge files to keep open
            max_len: the total length of strings stored before
                flushing counts to disk. This is an estimate of memory
                usage and not an accurate limit, as it doesn't include
                item counts or dict overhead.
        """

        self.max_fds = max_fds
        self.max_len = max_len

    def count(self, items):
        """Count string items.

        This method keeps an in-memory count of items until its size
        reaches max_len, then flushes the counts to a sorted overflow
        file on disk. It does this as many times as necessary, then
{'fraction_non_alphanumeric': 0.051444400474871387, 'fraction_numerical': 0.010288880094974277, 'mean_word_length': 2.5858156028368793, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4843697', 'n_tokens_mistral': 670, 'n_tokens_neox': 626, 'n_words': 308}