<commit_before>from itertools import chain, product
from thechronic.utils import is_iterable

class TheChronic(object):
    def __init__(self, words=[], files=[]):
        self._parse_words_arg(words)
        self._parse_files_arg(files)

    def combine(self, num_words=1, build_up=False):

        if build_up:
            iterators = ()
            for i in range(1, num_words + 1):
                res = product(self._words, repeat=i)
                iterators += (self._get_words_generator(res),)
            return chain(*iterators)

        res = product(self._words, repeat=num_words)

        return self._get_words_generator(res)

    def _get_words_generator(self, words):
        for word_parts in words:
            yield ''.join(word_parts)

    def _parse_words_arg(self, words):
        if is_iterable(words):
            if len(words) > 0 and is_iterable(words[0]):
                # we have an iterable of iterables (ex: a list of lists)
                self._words = list(chain.from_iterable(words))
            else:
                self._words = list(words)
        else:
            raise ValueError('Invalid argument for \'words\'.')

    def _parse_files_arg(self, files):
        if not is_iterable(files):
            files = (files,)
<commit_msg>feat: implement min/max output word length<commit_after>from itertools import chain, product
from thechronic.utils import is_iterable

class TheChronic(object):
    def __init__(self, words=[], files=[]):
        self._parse_words_arg(words)
        self._parse_files_arg(files)

    def combine(self, num_words=1, build_up=False, min_length=0, max_length=None):
        self._min_length = min_length
        self._max_length = max_length
        if build_up:
            iterators = ()
            for i in range(1, num_words + 1):
                res = product(self._words, repeat=i)
                iterators += (self._get_words_generator(res, min_length, max_length),)
            return chain(*iterators)

        res = product(self._words, repeat=num_words)
        return self._get_words_generator(res, min_length, max_length)

    def _get_words_generator(self, words, min_length, max_length):
        for word_parts in words:
            word = ''.join(word_parts)
            if self._is_word_within_limits(word, min_length, max_length):
                yield word

    def _is_word_within_limits(self, word, min_length, max_length):
        if len(word) >= min_length:
            if max_length is None:
                return True
            elif len(word) <= max_length:
                return True
        return False

    def _parse_words_arg(self, words):
        if is_iterable(words):
            if len(words) > 0 and is_iterable(words[0]):
                # we have an iterable of iterables (ex: a list of lists)
                self._words = list(chain.from_iterable(words))
            else:
                self._words = list(words)
        else:
            raise ValueError('Invalid argument for \'words\'.')

    def _parse_files_arg(self, files):
        if not is_iterable(files):
            files = (files,)

{'fraction_non_alphanumeric': 0.09572759396080951, 'fraction_numerical': 0.0035335689045936395, 'mean_word_length': 2.2539184952978055, 'pattern_counts': {'":': 0, '<': 4, '<?xml version=': 0, '>': 6, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '324256', 'n_tokens_mistral': 936, 'n_tokens_neox': 901, 'n_words': 226}