<commit_before>"""
Base classes

 .. note::

    All this class must be subclassed
"""
import sys
import logging

from pomp.core.utils import Planned


log = logging.getLogger('pomp.contrib.core')


class BaseCommand(object):
    pass


class BaseCrawler(object):
    """Crawler interface

    Crawler must resolve two main tasks:

    - Extract data from response
    - Extract next urls for following processing from response

    Each crawler must have starting point - entry url.
    To set entry url declare them as class attribute ``ENTRY_REQUESTS``
    like that::

        class MyGoogleCrawler(BaseCrawler):
            ENTRY_REQUESTS = 'http://google.com/'
            ...

    ``ENTRY_REQUESTS`` may be list of urls or list of requests
    (instances of :class:`BaseHttpRequest`).
    """
    ENTRY_REQUESTS = None

    def __init__(self):
        self._in_process = 0

    def next_requests(self, response):
        """Getting next requests for processing.

        Called after `extract_items` method.

        :note:
            Subclass can not implement thid method.
            Next requests may be returened with items in `extrat_items` method.

        :param response: the instance of :class:`BaseHttpResponse`
        :rtype: ``None`` or request or requests (instnace of
            :class:`BaseHttpRequest` or str). If ``None`` returned
            this mean that page have not any urls to following
            processing
        """
        pass

    def process(self, response):
        return self.extract_items(response)

    def extract_items(self, response):
        """Extract items and next requests - parse page.

        :param response: the instance of :class:`BaseHttpResponse`
        :rtype: item or items (isntance of :class:`pomp.core.item.Item`)
            or request or requests (instance of :class:`BaseHttpRequest`)
        """
        raise NotImplementedError()


class BaseQueue(object):
    """Blocking queue interface"""

    def get_requests(self):
        """Get from queue

        .. note::

            must block execution until item is available

        :rtype: instance of :class:`BaseRequest` or :class:`.Planned`
                or list of them
        """
        raise NotImplementedError()

    def put_requests(self, requests):
        """Put to queue

        :param requests: instance of :class:`BaseRequest` or list of them
        """
        raise NotImplementedError()


class BaseDownloadWorker(object):
    """Download worker interface"""

    def get_one(self, request):
        """Execute request

        :param request: instance of :class:`BaseHttpRequest`
        :rtype: instance of :class:`BaseHttpResponse` or
                :class:`BaseDownloadException` or :class:`.Planned`
                for async behavior
        """
        raise NotImplementedError()


class BaseDownloader(object):
    """Downloader interface

    Downloader must resolve one main task - execute request
    and fetch response.

    :param middlewares: list of middlewares, instances
                        of :class:`BaseDownloaderMiddleware`
    """

    def __init__(self, middlewares=None):
        self.middlewares = middlewares or []

        if isinstance(self.middlewares, tuple):
            self.middlewares = list(self.middlewares)

    def prepare(self):
        """Prepare downloader before start processing"""
        self.request_middlewares = self.middlewares
        self.response_middlewares = self.middlewares[:]
        self.response_middlewares.reverse()

    def process(self, to_process, crawler):
        if not to_process:
            return

        requests = []
        for request in to_process:

            for middleware in self.request_middlewares:
                try:
                    request = middleware.process_request(request)
                except Exception as e:
                    log.exception(
                        'Exception on process %s by %s', request, middleware)
                    yield self._process_exception(
                        BaseDownloadException(
                            request,
                            exception=e,
                            exc_info=sys.exc_info(),
                        )
                    )
                    request = None  # stop processing request by middlewares
                if not request:
                    break

            if not request:
                continue

            requests.append(request)

        if not requests:
            return

        for response in self.get(requests):
            if isinstance(response, Planned):  # async behavior
                future = Planned()

                def _chain(res):
                    from datetime import datetime
                    log.debug(
                        '%s BaseDownloader done: %s -> %s',
                        datetime.now(),
                        response, future,
                    )
                    future.set_result(self._process_resp(res.result()))

                response.add_done_callback(_chain)
                log.debug(
                    'BaseDownloader yield: %s wait: %s', future, response
                )
                yield future
            else:  # sync behavior
                yield self._process_resp(response)<commit_msg>cleanup logging<commit_after>"""
Base classes

 .. note::

    All this class must be subclassed
"""
import sys
import logging

from pomp.core.utils import Planned


log = logging.getLogger('pomp.contrib.core')


class BaseCommand(object):
    pass


class BaseCrawler(object):
    """Crawler interface

    Crawler must resolve two main tasks:

    - Extract data from response
    - Extract next urls for following processing from response

    Each crawler must have starting point - entry url.
    To set entry url declare them as class attribute ``ENTRY_REQUESTS``
    like that::

        class MyGoogleCrawler(BaseCrawler):
            ENTRY_REQUESTS = 'http://google.com/'
            ...

    ``ENTRY_REQUESTS`` may be list of urls or list of requests
    (instances of :class:`BaseHttpRequest`).
    """
    ENTRY_REQUESTS = None

    def __init__(self):
        self._in_process = 0

    def next_requests(self, response):
        """Getting next requests for processing.

        Called after `extract_items` method.

        :note:
            Subclass can not implement thid method.
            Next requests may be returened with items in `extrat_items` method.

        :param response: the instance of :class:`BaseHttpResponse`
        :rtype: ``None`` or request or requests (instnace of
            :class:`BaseHttpRequest` or str). If ``None`` returned
            this mean that page have not any urls to following
            processing
        """
        pass

    def process(self, response):
        return self.extract_items(response)

    def extract_items(self, response):
        """Extract items and next requests - parse page.

        :param response: the instance of :class:`BaseHttpResponse`
        :rtype: item or items (isntance of :class:`pomp.core.item.Item`)
            or request or requests (instance of :class:`BaseHttpRequest`)
        """
        raise NotImplementedError()


class BaseQueue(object):
    """Blocking queue interface"""

    def get_requests(self):
        """Get from queue

        .. note::

            must block execution until item is available

        :rtype: instance of :class:`BaseRequest` or :class:`.Planned`
                or list of them
        """
        raise NotImplementedError()

    def put_requests(self, requests):
        """Put to queue

        :param requests: instance of :class:`BaseRequest` or list of them
        """
        raise NotImplementedError()


class BaseDownloadWorker(object):
    """Download worker interface"""

    def get_one(self, request):
        """Execute request

        :param request: instance of :class:`BaseHttpRequest`
        :rtype: instance of :class:`BaseHttpResponse` or
                :class:`BaseDownloadException` or :class:`.Planned`
                for async behavior
        """
        raise NotImplementedError()


class BaseDownloader(object):
    """Downloader interface

    Downloader must resolve one main task - execute request
    and fetch response.

    :param middlewares: list of middlewares, instances
                        of :class:`BaseDownloaderMiddleware`
    """

    def __init__(self, middlewares=None):
        self.middlewares = middlewares or []

        if isinstance(self.middlewares, tuple):
            self.middlewares = list(self.middlewares)

    def prepare(self):
        """Prepare downloader before start processing"""
        self.request_middlewares = self.middlewares
        self.response_middlewares = self.middlewares[:]
        self.response_middlewares.reverse()

    def process(self, to_process, crawler):
        if not to_process:
            return

        requests = []
        for request in to_process:

            for middleware in self.request_middlewares:
                try:
                    request = middleware.process_request(request)
                except Exception as e:
                    log.exception(
                        'Exception on process %s by %s', request, middleware)
                    yield self._process_exception(
                        BaseDownloadException(
                            request,
                            exception=e,
                            exc_info=sys.exc_info(),
                        )
                    )
                    request = None  # stop processing request by middlewares
                if not request:
                    break

            if not request:
                continue

            requests.append(request)

        if not requests:
            return

        for response in self.get(requests):
            if isinstance(response, Planned):  # async behavior
                future = Planned()

                def _chain(res):
                    future.set_result(self._process_resp(res.result()))

                response.add_done_callback(_chain)
                yield future
            else:  # sync behavior
                yield self._process_resp(response)
{'fraction_non_alphanumeric': 0.08025291828793775, 'fraction_numerical': 0.0001945525291828794, 'mean_word_length': 1.8590100111234704, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 4, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 20, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3051746', 'n_tokens_mistral': 2608, 'n_tokens_neox': 2452, 'n_words': 797}