<commit_before>"""
This module contains essential stuff that should've come with Python itself ;)
"""
import os
import re
import inspect
import weakref
import errno
import six
from functools import partial, wraps

from scrapy.utils.decorators import deprecated


def flatten(x):
    """flatten(sequence) -> list

    Returns a single, flat list which contains all elements retrieved
    from the sequence and all recursively contained sub-sequences
    (iterables).

    Examples:
    >>> [1, 2, [3,4], (5,6)]
    [1, 2, [3, 4], (5, 6)]
    >>> flatten([[[1,2,3], (42,None)], [4,5], [6], 7, (8,9,10)])
    [1, 2, 3, 42, None, 4, 5, 6, 7, 8, 9, 10]"""

    return list(iflatten(x))


def iflatten(x):
    """iflatten(sequence) -> iterator

    Similar to ``.flatten()``, but returns iterator instead"""

    for el in x:
        if hasattr(el, "__iter__"):
            for el_ in flatten(el):
                yield el_
        else:
            yield el


def unique(list_, key=lambda x: x):
    """efficient function to uniquify a list preserving item order"""
    seen = set()
    result = []
    for item in list_:
        seenkey = key(item)
        if seenkey in seen:
            continue
        seen.add(seenkey)
        result.append(item)
    return result


@deprecated("scrapy.utils.python.to_unicode")
def str_to_unicode(text, encoding=None, errors='strict'):
    """ This function is deprecated.
    Please use scrapy.utils.python.to_unicode. """
    return to_unicode(text, encoding, errors)<commit_msg>PY3 port flatten and iflatten<commit_after>"""
This module contains essential stuff that should've come with Python itself ;)
"""
import os
import re
import inspect
import weakref
import errno
import six
from functools import partial, wraps

from scrapy.utils.decorators import deprecated


def flatten(x):
    """flatten(sequence) -> list

    Returns a single, flat list which contains all elements retrieved
    from the sequence and all recursively contained sub-sequences
    (iterables).

    Examples:
    >>> [1, 2, [3,4], (5,6)]
    [1, 2, [3, 4], (5, 6)]
    >>> flatten([[[1,2,3], (42,None)], [4,5], [6], 7, (8,9,10)])
    [1, 2, 3, 42, None, 4, 5, 6, 7, 8, 9, 10]
    >>> flatten(["foo", "bar"])
    ['foo', 'bar']
    >>> flatten(["foo", ["baz", 42], "bar"])
    ['foo', 'baz', 42, 'bar']
    """
    return list(iflatten(x))


def iflatten(x):
    """iflatten(sequence) -> iterator

    Similar to ``.flatten()``, but returns iterator instead"""
    for el in x:
        if is_listlike(el):
            for el_ in flatten(el):
                yield el_
        else:
            yield el


def is_listlike(x):
    """
    >>> is_listlike("foo")
    False
    >>> is_listlike(5)
    False
    >>> is_listlike(b"foo")
    False
    >>> is_listlike([b"foo"])
    True
    >>> is_listlike((b"foo",))
    True
    >>> is_listlike({})
    True
    >>> is_listlike(set())
    True
    >>> is_listlike((x for x in range(3)))
    True
    >>> is_listlike(six.moves.xrange(5))
    True
    """
    return hasattr(x, "__iter__") and not isinstance(x, (six.text_type, bytes))


def unique(list_, key=lambda x: x):
    """efficient function to uniquify a list preserving item order"""
    seen = set()
    result = []
    for item in list_:
        seenkey = key(item)
        if seenkey in seen:
            continue
        seen.add(seenkey)
        result.append(item)
    return result


@deprecated("scrapy.utils.python.to_unicode")
def str_to_unicode(text, encoding=None, errors='strict'):
    """ This function is deprecated.
    Please use scrapy.utils.python.to_unicode. """
    return to_unicode(text, encoding, errors)
{'fraction_non_alphanumeric': 0.14642171647929805, 'fraction_numerical': 0.023032629558541268, 'mean_word_length': 2.8603174603174604, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 52, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5750949', 'n_tokens_mistral': 1266, 'n_tokens_neox': 1174, 'n_words': 370}