<commit_before>import logging
import os
import sys
import tempfile
import time

import requests
from .cache import CachingSession, FileCache    # noqa

if sys.version_info[0] < 3:         # pragma: no cover
    from urllib2 import urlopen as urllib_urlopen
    from urllib2 import URLError as urllib_URLError
    _str_type = unicode
else:                               # pragma: no cover
    from urllib.request import urlopen as urllib_urlopen
    from urllib.error import URLError as urllib_URLError
    _str_type = str

__version__ = '1.1.0'
_user_agent = ' '.join(('scrapelib', __version__, requests.utils.default_user_agent()))


class NullHandler(logging.Handler):
    def emit(self, record):
        pass

_log = logging.getLogger('scrapelib')
_log.addHandler(NullHandler())


class HTTPMethodUnavailableError(requests.RequestException):
    """
    Raised when the supplied HTTP method is invalid or not supported
    by the HTTP backend.
    """

    def __init__(self, message, method):
        super(HTTPMethodUnavailableError, self).__init__(message)
        self.method = method


class HTTPError(requests.HTTPError):
    """
    Raised when urlopen encounters a 4xx or 5xx error code and the
    raise_errors option is true.
    """

    def __init__(self, response, body=None):
        message = '%s while retrieving %s' % (response.status_code, response.url)
        super(HTTPError, self).__init__(message)
        self.response = response
        self.body = body or self.response.text


class FTPError(requests.HTTPError):
    def __init__(self, url):
        message = 'error while retrieving %s' % url
        super(FTPError, self).__init__(message)


class ThrottledSession(requests.Session):
    def _throttle(self):
        now = time.time()
        diff = self._request_frequency - (now - self._last_request)
        if diff > 0:
            _log.debug("sleeping for %fs" % diff)
            time.sleep(diff)
            self._last_request = time.time()
        else:
            self._last_request = now

    @property
    def requests_per_minute(self):
        return self._requests_per_minute

    @requests_per_minute.setter
    def requests_per_minute(self, value):
        if value > 0:
            self._throttled = True
            self._requests_per_minute = value
            self._request_frequency = 60.0 / value
            self._last_request = 0
        else:
            self._throttled = False
            self._requests_per_minute = 0
            self._request_frequency = 0.0
            self._last_request = 0

    def request(self, method, url, **kwargs):
        if self._throttled:
            self._throttle()
        return super(ThrottledSession, self).request(method, url, **kwargs)


# this object exists because Requests assumes it can call
# resp.raw._original_response.msg.getheaders() and we need to cope with that
class DummyObject(object):
    def getheaders(self, name):
        return ''

    def get_all(self, name, default):
        return default

_dummy = DummyObject()
_dummy._original_response = DummyObject()
_dummy._original_response.msg = DummyObject()


class FTPAdapter(requests.adapters.BaseAdapter):

    def send(self, request, stream=False, timeout=None, verify=False, cert=None, proxies=None):
        if request.method != 'GET':
            raise HTTPMethodUnavailableError("FTP requests do not support method '%s'" %
                                             request.method, request.method)
        try:
            real_resp = urllib_urlopen(request.url, timeout=timeout)
            # we're going to fake a requests.Response with this
            resp = requests.Response()
            resp.status_code = 200
            resp.url = request.url
            resp.headers = {}
            resp._content = real_resp.read()
            resp.raw = _dummy
            return resp
        except urllib_URLError:
            raise FTPError(request.url)


class RetrySession(requests.Session):

    def __init__(self):
        super(RetrySession, self).__init__()
        self._retry_attempts = 0
        self.retry_wait_seconds = 10

    # retry_attempts is a property so that it can't go negative
    @property
    def retry_attempts(self):
        return self._retry_attempts

    @retry_attempts.setter
    def retry_attempts(self, value):
        self._retry_attempts = max(value, 0)

    def accept_response(self, response, **kwargs):
        return response.status_code < 400

    def request(self, method, url, retry_on_404=False, **kwargs):
        # the retry loop
        tries = 0
        exception_raised = None

        while tries <= self.retry_attempts:
            exception_raised = None

            try:
                resp = super(RetrySession, self).request(method, url, **kwargs)
                # break from loop on an accepted response
                if self.accept_response(resp) or (resp.status_code == 404 and not retry_on_404):
                    break

            # note: This is a pretty broad catch-all, but given the plethora of things that can
            #       happen during a requests.request it is used to try to be complete &
            #       future-proof this as much as possible.
            #       Should it become a problem we could either alter to exclude a few others
            #       the way we handle SSLError or we could go back to enumeration of all types.
            except Exception as e:
                if isinstance(e, requests.exceptions.SSLError):
                    raise
                exception_raised = e

            # if we're going to retry, sleep first
            tries += 1
            if tries <= self.retry_attempts:
                # twice as long each time
                wait = (self.retry_wait_seconds * (2 ** (tries - 1)))
                _log.debug('sleeping for %s seconds before retry' % wait)
                time.sleep(wait)

        # out of the loop, either an exception was raised or we had a success
        if exception_raised:
            raise exception_raised
        else:<commit_msg>retries should be warnings<commit_after>import logging
import os
import sys
import tempfile
import time

import requests
from .cache import CachingSession, FileCache    # noqa

if sys.version_info[0] < 3:         # pragma: no cover
    from urllib2 import urlopen as urllib_urlopen
    from urllib2 import URLError as urllib_URLError
    _str_type = unicode
else:                               # pragma: no cover
    from urllib.request import urlopen as urllib_urlopen
    from urllib.error import URLError as urllib_URLError
    _str_type = str

__version__ = '1.1.0'
_user_agent = ' '.join(('scrapelib', __version__, requests.utils.default_user_agent()))


_log = logging.getLogger('scrapelib')
_log.addHandler(logging.NullHandler())


class HTTPMethodUnavailableError(requests.RequestException):
    """
    Raised when the supplied HTTP method is invalid or not supported
    by the HTTP backend.
    """

    def __init__(self, message, method):
        super(HTTPMethodUnavailableError, self).__init__(message)
        self.method = method


class HTTPError(requests.HTTPError):
    """
    Raised when urlopen encounters a 4xx or 5xx error code and the
    raise_errors option is true.
    """

    def __init__(self, response, body=None):
        message = '%s while retrieving %s' % (response.status_code, response.url)
        super(HTTPError, self).__init__(message)
        self.response = response
        self.body = body or self.response.text


class FTPError(requests.HTTPError):
    def __init__(self, url):
        message = 'error while retrieving %s' % url
        super(FTPError, self).__init__(message)


class ThrottledSession(requests.Session):
    def _throttle(self):
        now = time.time()
        diff = self._request_frequency - (now - self._last_request)
        if diff > 0:
            _log.debug("sleeping for %fs" % diff)
            time.sleep(diff)
            self._last_request = time.time()
        else:
            self._last_request = now

    @property
    def requests_per_minute(self):
        return self._requests_per_minute

    @requests_per_minute.setter
    def requests_per_minute(self, value):
        if value > 0:
            self._throttled = True
            self._requests_per_minute = value
            self._request_frequency = 60.0 / value
            self._last_request = 0
        else:
            self._throttled = False
            self._requests_per_minute = 0
            self._request_frequency = 0.0
            self._last_request = 0

    def request(self, method, url, **kwargs):
        if self._throttled:
            self._throttle()
        return super(ThrottledSession, self).request(method, url, **kwargs)


# this object exists because Requests assumes it can call
# resp.raw._original_response.msg.getheaders() and we need to cope with that
class DummyObject(object):
    def getheaders(self, name):
        return ''

    def get_all(self, name, default):
        return default

_dummy = DummyObject()
_dummy._original_response = DummyObject()
_dummy._original_response.msg = DummyObject()


class FTPAdapter(requests.adapters.BaseAdapter):

    def send(self, request, stream=False, timeout=None, verify=False, cert=None, proxies=None):
        if request.method != 'GET':
            raise HTTPMethodUnavailableError("FTP requests do not support method '%s'" %
                                             request.method, request.method)
        try:
            real_resp = urllib_urlopen(request.url, timeout=timeout)
            # we're going to fake a requests.Response with this
            resp = requests.Response()
            resp.status_code = 200
            resp.url = request.url
            resp.headers = {}
            resp._content = real_resp.read()
            resp.raw = _dummy
            return resp
        except urllib_URLError:
            raise FTPError(request.url)


class RetrySession(requests.Session):

    def __init__(self):
        super(RetrySession, self).__init__()
        self._retry_attempts = 0
        self.retry_wait_seconds = 10

    # retry_attempts is a property so that it can't go negative
    @property
    def retry_attempts(self):
        return self._retry_attempts

    @retry_attempts.setter
    def retry_attempts(self, value):
        self._retry_attempts = max(value, 0)

    def accept_response(self, response, **kwargs):
        return response.status_code < 400

    def request(self, method, url, retry_on_404=False, **kwargs):
        # the retry loop
        tries = 0
        exception_raised = None

        while tries <= self.retry_attempts:
            exception_raised = None

            try:
                resp = super(RetrySession, self).request(method, url, **kwargs)
                # break from loop on an accepted response
                if self.accept_response(resp) or (resp.status_code == 404 and not retry_on_404):
                    break

            # note: This is a pretty broad catch-all, but given the plethora of things that can
            #       happen during a requests.request it is used to try to be complete &
            #       future-proof this as much as possible.
            #       Should it become a problem we could either alter to exclude a few others
            #       the way we handle SSLError or we could go back to enumeration of all types.
            except Exception as e:
                if isinstance(e, requests.exceptions.SSLError):
                    raise
                exception_raised = e

            # if we're going to retry, sleep first
            tries += 1
            if tries <= self.retry_attempts:
                # twice as long each time
                wait = (self.retry_wait_seconds * (2 ** (tries - 1)))
                _log.debug('sleeping for %s seconds before retry' % wait)
                if exception_raised:
                    _log.warn('got %s sleeping for %s seconds before retry', exception_raised, wait)
                else:
                    _log.warn('sleeping for %s seconds before retry', wait)
                time.sleep(wait)

        # out of the loop, either an exception was raised or we had a success
        if exception_raised:
            raise exception_raised
        else:
{'fraction_non_alphanumeric': 0.07926184863019267, 'fraction_numerical': 0.006828713112755061, 'mean_word_length': 2.209496477954605, 'pattern_counts': {'":': 0, '<': 11, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 6, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1963338', 'n_tokens_mistral': 3514, 'n_tokens_neox': 3345, 'n_words': 1014}