<commit_before>""" An extractor is a callable that returns or yields data. For example:

.. code-block:: python

    def extract(response):
        return "something"

The ``response`` parameter here is an instance of 
:class:`wex.response.Response`.

Extractors can be combined in various ways.
"""

from __future__ import absolute_import, unicode_literals, print_function
from functools import wraps
from .value import yield_values


OMITTED = object()


def chained(*extractors):
    """ Returns an extractor that chains the output of other extractors.

    :param extractors: an iterable of extractor callables to chain

    For example an extractor function ``extract`` defined as follows:

    .. code-block:: python

        def extract1(response):
            yield "one"

        def extract2(response):
            yield "two"

        extract = chained(extract1, extract2)

    Would produce the following extraction output:

    .. code-block:: shell

        $ wex http://example.net/
        "one"
        "two"

    """
    return ChainedExtractors(extractors)


class ChainedExtractors(object):

    @property
    def __name__(self):
        return repr(self)

    def __repr__(self):
        return '%s(%r)' % (self.__class__.__name__, self.extractors)

    def __init__(self, extractors):
        self.extractors = extractors

    def __call__(self, arg0, *args, **kw):
        seek = getattr(arg0, 'seek', None)
        for extractor in self.extractors:
            if seek:
                seek(0)
            values = yield_values(extractor, arg0, *args, **kw)
            for value in values:
                yield value


def labelled(*literals_or_callables):
    """ Returns a decorator that will label the output an extractor.

    :param literals_or_callables: An iterable of labels or callables.

    Each item in ``literals_or_callables`` may be a literal or a callable.
    Any callable will called with the same parameters as the extractor
    and whatever is returned will by used as a label.

    For example an extractor function ``extract`` defined as follows:

    .. code-block:: python

        def extract1(response):
            yield "one"


        def label2(response):
            return "label2"


        extract = labelled("label1", label2)(extract1)

    Would produce the following extraction output:

    .. code-block:: shell

        $ wex http://example.net/
        "label1"    "label2"    "one"

    Note that if any of the labels are
    `false <https://docs.python.org/2/library/stdtypes.html#truth-value-testing>`_ 
    then no output will be generated from that extractor.
    """

    def call(label, arg0):
        return (label(arg0) if hasattr(label, '__call__') else label)

    def labelled_extractor_decorator(extractor):
        @wraps(extractor)
        def labelled_extractor_wrapper(arg0, *args, **kw):

            labels = [call(label, arg0) for label in literals_or_callables]
            if not all(labels):
                # one or more missing labels so don't yield
                return

            for value in yield_values(extractor, arg0, *args, **kw):
                yield value.label(*labels)

        return labelled_extractor_wrapper

    return labelled_extractor_decorator


class Attributes(object):
    """ A extractor that is a collection of labelled extractors.

    Extractors can be added to the collection on construction
    using keyword arguments for the labels.  For example, an 
    extractor function ``extract`` defined as follows:

    .. code-block:: python

        extract = Attributes(<commit_msg>Improved decorator name<commit_after>""" An extractor is a callable that returns or yields data. For example:

.. code-block:: python

    def extract(response):
        return "something"

The ``response`` parameter here is an instance of 
:class:`wex.response.Response`.

Extractors can be combined in various ways.
"""

from __future__ import absolute_import, unicode_literals, print_function
from functools import wraps
from .value import yield_values


OMITTED = object()


def chained(*extractors):
    """ Returns an extractor that chains the output of other extractors.

    :param extractors: an iterable of extractor callables to chain

    For example an extractor function ``extract`` defined as follows:

    .. code-block:: python

        def extract1(response):
            yield "one"

        def extract2(response):
            yield "two"

        extract = chained(extract1, extract2)

    Would produce the following extraction output:

    .. code-block:: shell

        $ wex http://example.net/
        "one"
        "two"

    """
    return ChainedExtractors(extractors)


class ChainedExtractors(object):

    @property
    def __name__(self):
        return repr(self)

    def __repr__(self):
        return '%s(%r)' % (self.__class__.__name__, self.extractors)

    def __init__(self, extractors):
        self.extractors = extractors

    def __call__(self, arg0, *args, **kw):
        seek = getattr(arg0, 'seek', None)
        for extractor in self.extractors:
            if seek:
                seek(0)
            values = yield_values(extractor, arg0, *args, **kw)
            for value in values:
                yield value


def labelled(*literals_or_callables):
    """ Returns a decorator that will label the output an extractor.

    :param literals_or_callables: An iterable of labels or callables.

    Each item in ``literals_or_callables`` may be a literal or a callable.
    Any callable will called with the same parameters as the extractor
    and whatever is returned will by used as a label.

    For example an extractor function ``extract`` defined as follows:

    .. code-block:: python

        def extract1(response):
            yield "one"


        def label2(response):
            return "label2"


        extract = labelled("label1", label2)(extract1)

    Would produce the following extraction output:

    .. code-block:: shell

        $ wex http://example.net/
        "label1"    "label2"    "one"

    Note that if any of the labels are
    `false <https://docs.python.org/2/library/stdtypes.html#truth-value-testing>`_ 
    then no output will be generated from that extractor.
    """

    def call(label, arg0):
        return (label(arg0) if hasattr(label, '__call__') else label)

    def labelled_extractor_decorator(extractor):
        @wraps(extractor)
        def labelled_extractor(arg0, *args, **kw):
            labels = [call(label, arg0) for label in literals_or_callables]
            if not all(labels):
                # one or more missing labels so don't yield
                return

            for value in yield_values(extractor, arg0, *args, **kw):
                yield value.label(*labels)

        return labelled_extractor

    return labelled_extractor_decorator


class Attributes(object):
    """ A extractor that is a collection of labelled extractors.

    Extractors can be added to the collection on construction
    using keyword arguments for the labels.  For example, an 
    extractor function ``extract`` defined as follows:

    .. code-block:: python

        extract = Attributes(
{'fraction_non_alphanumeric': 0.08637689209831968, 'fraction_numerical': 0.00611026246354673, 'mean_word_length': 2.5866533864541834, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1499083', 'n_tokens_mistral': 2053, 'n_tokens_neox': 1929, 'n_words': 695}