<commit_before>import os
import sys
import time
import socket
import logging
import tempfile
import datetime
import json
import httplib2

if sys.version_info[0] < 3:
    PY3K = False
    from urllib2 import Request as urllib_Request
    from urllib2 import urlopen as urllib_urlopen
    from urllib2 import URLError as urllib_URLError
    import urlparse
    import robotparser
    from cookielib import CookieJar
    _str_type = unicode
    _str_types = basestring
else:
    PY3K = True
    from urllib.request import Request as urllib_Request
    from urllib.request import urlopen as urllib_urlopen
    from urllib.error import URLError as urllib_URLError
    from urllib import parse as urlparse
    from urllib import robotparser
    from http.cookiejar import CookieJar
    _str_type = str
    _str_types = (str, bytes)

__version__ = '0.6.0-dev'
_user_agent = 'scrapelib %s' % __version__


class NullHandler(logging.Handler):
    def emit(self, record):
        pass

_log = logging.getLogger('scrapelib')
_log.addHandler(NullHandler())


class ScrapeError(Exception):
    pass


class RobotExclusionError(ScrapeError):
    """
    Raised when an attempt is made to access a page denied by
    the host's robots.txt file.
    """

    def __init__(self, message, url, user_agent):
        super(RobotExclusionError, self).__init__(message)
        self.url = url
        self.user_agent = user_agent


class HTTPMethodUnavailableError(ScrapeError):
    """
    Raised when the supplied HTTP method is invalid or not supported
    by the HTTP backend.
    """

    def __init__(self, message, method):
        super(HTTPMethodUnavailableError, self).__init__(message)
        self.method = method


class HTTPError(ScrapeError):
    """
    Raised when urlopen encounters a 4xx or 5xx error code and the
    raise_errors option is true.
    """

    def __init__(self, response, body):
        message = '%s while retrieving %s' % (response.code, response.url)
        super(HTTPError, self).__init__(message)
        self.response = response
        self.body = body


class ErrorManager(object):
    def __enter__(self):
        return self

    def __exit__(self, exc_type, exc_val, exc_tb):
        if exc_type and self._scraper.save_errors:
            self._scraper._save_error(self.response.url, self)
        return False


class ResultStr(_str_type, ErrorManager):
    """
    Wrapper for responses.  Can treat identically to a ``str``
    o get body of response, additional headers, etc. available via ``response``
    attribute (instance of :class:`Response`).
    """
    def __new__(cls, scraper, response, s):
        self = _str_type.__new__(cls, s)
        self._scraper = scraper
        self.response = response
        return self
ResultUnicode = ResultStr


class Headers(dict):
    """
    Dictionary-like object for storing response headers in a
    case-insensitive way (keeping with the HTTP spec).

    Accessed as the ``headers`` attribute of :class:`Response`.<commit_msg>if bytes, decode()<commit_after>import os
import sys
import time
import socket
import logging
import tempfile
import datetime
import json
import httplib2

if sys.version_info[0] < 3:
    PY3K = False
    from urllib2 import Request as urllib_Request
    from urllib2 import urlopen as urllib_urlopen
    from urllib2 import URLError as urllib_URLError
    import urlparse
    import robotparser
    from cookielib import CookieJar
    _str_type = unicode
    _str_types = basestring
else:
    PY3K = True
    from urllib.request import Request as urllib_Request
    from urllib.request import urlopen as urllib_urlopen
    from urllib.error import URLError as urllib_URLError
    from urllib import parse as urlparse
    from urllib import robotparser
    from http.cookiejar import CookieJar
    _str_type = str
    _str_types = (str, bytes)

__version__ = '0.6.0-dev'
_user_agent = 'scrapelib %s' % __version__


class NullHandler(logging.Handler):
    def emit(self, record):
        pass

_log = logging.getLogger('scrapelib')
_log.addHandler(NullHandler())


class ScrapeError(Exception):
    pass


class RobotExclusionError(ScrapeError):
    """
    Raised when an attempt is made to access a page denied by
    the host's robots.txt file.
    """

    def __init__(self, message, url, user_agent):
        super(RobotExclusionError, self).__init__(message)
        self.url = url
        self.user_agent = user_agent


class HTTPMethodUnavailableError(ScrapeError):
    """
    Raised when the supplied HTTP method is invalid or not supported
    by the HTTP backend.
    """

    def __init__(self, message, method):
        super(HTTPMethodUnavailableError, self).__init__(message)
        self.method = method


class HTTPError(ScrapeError):
    """
    Raised when urlopen encounters a 4xx or 5xx error code and the
    raise_errors option is true.
    """

    def __init__(self, response, body):
        message = '%s while retrieving %s' % (response.code, response.url)
        super(HTTPError, self).__init__(message)
        self.response = response
        self.body = body


class ErrorManager(object):
    def __enter__(self):
        return self

    def __exit__(self, exc_type, exc_val, exc_tb):
        if exc_type and self._scraper.save_errors:
            self._scraper._save_error(self.response.url, self)
        return False


class ResultStr(_str_type, ErrorManager):
    """
    Wrapper for responses.  Can treat identically to a ``str``
    o get body of response, additional headers, etc. available via ``response``
    attribute (instance of :class:`Response`).
    """
    def __new__(cls, scraper, response, s):
        if isinstance(s, bytes):
            s = s.decode()
        self = _str_type.__new__(cls, s)
        self._scraper = scraper
        self.response = response
        return self
ResultUnicode = ResultStr


class Headers(dict):
    """
    Dictionary-like object for storing response headers in a
    case-insensitive way (keeping with the HTTP spec).

    Accessed as the ``headers`` attribute of :class:`Response`.
{'fraction_non_alphanumeric': 0.07717623827546487, 'fraction_numerical': 0.004278426855356262, 'mean_word_length': 3.145975443383356, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7096564', 'n_tokens_mistral': 1876, 'n_tokens_neox': 1801, 'n_words': 578}