<commit_before>"""
Copyright (c) 2015 Red Hat, Inc
All rights reserved.

This software may be modified and distributed under the terms
of the BSD license. See the LICENSE file for details.


abstraction on top of http api calls

use pycurl (can handle chunked response properly), fallback to requests

chunked implementation for pycurl taken from:
  http://stackoverflow.com/a/21809888/909579
"""

from __future__ import print_function, absolute_import, unicode_literals

import re
import json
import logging
import email.parser
from osbs.exceptions import OsbsException, OsbsNetworkException

try:
    # py2
    import httplib
    from urllib2 import HTTPError
    from StringIO import StringIO as BytesIO
except ImportError:
    # py3
    import http.client as httplib
    from urllib.error import HTTPError
    from io import BytesIO


logger = logging.getLogger(__name__)

# requests_imported = False
# pycurl_imported = False

# prefered http lib is pycurl, since it understands chunked responses and kerberos
import pycurl
pycurl_imported = True

# FIXME: fix compat abstraction between requests and pycurl
#        so core doesn't have to care about chosen http lib

# try:
#     import pycurl
# except ImportError:
#     try:
#         import requests
#     except ImportError:
#         print("Neither requests, nor pycurl are available.")
#         sys.exit(1)
#     else:
#         requests_imported = True
# else:
#     pycurl_imported = True


SELECT_TIMEOUT = 9999
PYCURL_NETWORK_CODES = [pycurl.E_BAD_CONTENT_ENCODING,
                        pycurl.E_BAD_DOWNLOAD_RESUME,
                        pycurl.E_CONV_FAILED,
                        pycurl.E_CONV_REQD,
                        pycurl.E_COULDNT_CONNECT,
                        pycurl.E_COULDNT_RESOLVE_HOST,
                        pycurl.E_COULDNT_RESOLVE_PROXY,
                        pycurl.E_FILESIZE_EXCEEDED,
                        pycurl.E_HTTP_POST_ERROR,
                        pycurl.E_HTTP_RANGE_ERROR,
                        pycurl.E_HTTP_RETURNED_ERROR,
                        pycurl.E_LOGIN_DENIED,
                        # old pycurl: E_OPERATION_TIMEOUTED, new pycurl: E_OPERATION_TIMEDOUT
                        getattr(pycurl, "E_OPERATION_TIMEDOUT", "E_OPERATION_TIMEOUTED"),
                        pycurl.E_PARTIAL_FILE,
                        pycurl.E_READ_ERROR,
                        pycurl.E_RECV_ERROR,
                        pycurl.E_REMOTE_FILE_NOT_FOUND,
                        pycurl.E_SEND_ERROR,
                        pycurl.E_SSL_CACERT,
                        pycurl.E_SSL_CERTPROBLEM,
                        pycurl.E_SSL_CIPHER,
                        pycurl.E_SSL_CONNECT_ERROR,
                        pycurl.E_SSL_PEER_CERTIFICATE,
                        pycurl.E_SSL_SHUTDOWN_FAILED,
                        pycurl.E_TOO_MANY_REDIRECTS,
                        pycurl.E_UNSUPPORTED_PROTOCOL,
                        pycurl.E_WRITE_ERROR]

PYCURL_NETWORK_CODES = [x for x in PYCURL_NETWORK_CODES if x is not None]


class Response(object):
    """ let's mock Response object of requests """

    def __init__(self, status_code=0, content=b'', curl=None, raw_headers=None):
        self.status_code = status_code
        self.content = content
        self.curl = curl
        self.curl_multi = getattr(self.curl, "curl_multi", None)
        self.response_buffer = getattr(self.curl, "response", None)
        self._raw_headers = raw_headers
        self._headers = None

    @property
    def raw_headers(self):
        """
        Headers of request (str)
        """
        return self._raw_headers

    @raw_headers.setter
    def raw_headers(self, raw_headers):
        self._raw_headers = raw_headers

    @property
    def headers(self):
        if self._headers is None:
            logger.debug("raw headers: " + repr(self.raw_headers))
            encoded_raw_headers = self.raw_headers.encode('iso-8859-1')
            headers_buffer = BytesIO(encoded_raw_headers)
            try:
                # py 2
                # seekable has to be 0, otherwise it won't parse anything
                m = httplib.HTTPMessage(headers_buffer, seekable=0)
                m.readheaders()
                self._headers = m.dict
            except TypeError as ex:
                # py 3
                if ex.args[0] == "__init__() got an unexpected keyword argument 'seekable'":
                    parser = email.parser.Parser()
                    m = parser.parsestr(encoded_raw_headers)
                    self._headers = dict(m.items())
                else:
                    raise
        return self._headers

    @property
    def encoding(self):
        encoding = None
        if 'content-type' in self.headers:
            content_type = self.headers['content-type'].lower()
            match = re.search(r'charset=(\S+)', content_type)
            if match:
                encoding = match.group(1)
        if encoding is None:
            encoding = 'utf-8'  # assume utf-8

        return encoding

    def json(self, check=True):
        if check:
            self._check_status_code()
        return json.loads(self.content.decode(self.encoding))<commit_msg>Print http body if check_status_code raised HTTPError<commit_after>"""
Copyright (c) 2015 Red Hat, Inc
All rights reserved.

This software may be modified and distributed under the terms
of the BSD license. See the LICENSE file for details.


abstraction on top of http api calls

use pycurl (can handle chunked response properly), fallback to requests

chunked implementation for pycurl taken from:
  http://stackoverflow.com/a/21809888/909579
"""

from __future__ import print_function, absolute_import, unicode_literals

import re
import json
import logging
import email.parser
from osbs.exceptions import OsbsException, OsbsNetworkException

try:
    # py2
    import httplib
    from urllib2 import HTTPError
    from StringIO import StringIO as BytesIO
except ImportError:
    # py3
    import http.client as httplib
    from urllib.error import HTTPError
    from io import BytesIO


logger = logging.getLogger(__name__)

# requests_imported = False
# pycurl_imported = False

# prefered http lib is pycurl, since it understands chunked responses and kerberos
import pycurl
pycurl_imported = True

# FIXME: fix compat abstraction between requests and pycurl
#        so core doesn't have to care about chosen http lib

# try:
#     import pycurl
# except ImportError:
#     try:
#         import requests
#     except ImportError:
#         print("Neither requests, nor pycurl are available.")
#         sys.exit(1)
#     else:
#         requests_imported = True
# else:
#     pycurl_imported = True


SELECT_TIMEOUT = 9999
PYCURL_NETWORK_CODES = [pycurl.E_BAD_CONTENT_ENCODING,
                        pycurl.E_BAD_DOWNLOAD_RESUME,
                        pycurl.E_CONV_FAILED,
                        pycurl.E_CONV_REQD,
                        pycurl.E_COULDNT_CONNECT,
                        pycurl.E_COULDNT_RESOLVE_HOST,
                        pycurl.E_COULDNT_RESOLVE_PROXY,
                        pycurl.E_FILESIZE_EXCEEDED,
                        pycurl.E_HTTP_POST_ERROR,
                        pycurl.E_HTTP_RANGE_ERROR,
                        pycurl.E_HTTP_RETURNED_ERROR,
                        pycurl.E_LOGIN_DENIED,
                        # old pycurl: E_OPERATION_TIMEOUTED, new pycurl: E_OPERATION_TIMEDOUT
                        getattr(pycurl, "E_OPERATION_TIMEDOUT", "E_OPERATION_TIMEOUTED"),
                        pycurl.E_PARTIAL_FILE,
                        pycurl.E_READ_ERROR,
                        pycurl.E_RECV_ERROR,
                        pycurl.E_REMOTE_FILE_NOT_FOUND,
                        pycurl.E_SEND_ERROR,
                        pycurl.E_SSL_CACERT,
                        pycurl.E_SSL_CERTPROBLEM,
                        pycurl.E_SSL_CIPHER,
                        pycurl.E_SSL_CONNECT_ERROR,
                        pycurl.E_SSL_PEER_CERTIFICATE,
                        pycurl.E_SSL_SHUTDOWN_FAILED,
                        pycurl.E_TOO_MANY_REDIRECTS,
                        pycurl.E_UNSUPPORTED_PROTOCOL,
                        pycurl.E_WRITE_ERROR]

PYCURL_NETWORK_CODES = [x for x in PYCURL_NETWORK_CODES if x is not None]


class Response(object):
    """ let's mock Response object of requests """

    def __init__(self, status_code=0, content=b'', curl=None, raw_headers=None):
        self.status_code = status_code
        self.content = content
        self.curl = curl
        self.curl_multi = getattr(self.curl, "curl_multi", None)
        self.response_buffer = getattr(self.curl, "response", None)
        self._raw_headers = raw_headers
        self._headers = None

    @property
    def raw_headers(self):
        """
        Headers of request (str)
        """
        return self._raw_headers

    @raw_headers.setter
    def raw_headers(self, raw_headers):
        self._raw_headers = raw_headers

    @property
    def headers(self):
        if self._headers is None:
            logger.debug("raw headers: " + repr(self.raw_headers))
            encoded_raw_headers = self.raw_headers.encode('iso-8859-1')
            headers_buffer = BytesIO(encoded_raw_headers)
            try:
                # py 2
                # seekable has to be 0, otherwise it won't parse anything
                m = httplib.HTTPMessage(headers_buffer, seekable=0)
                m.readheaders()
                self._headers = m.dict
            except TypeError as ex:
                # py 3
                if ex.args[0] == "__init__() got an unexpected keyword argument 'seekable'":
                    parser = email.parser.Parser()
                    m = parser.parsestr(encoded_raw_headers)
                    self._headers = dict(m.items())
                else:
                    raise
        return self._headers

    @property
    def encoding(self):
        encoding = None
        if 'content-type' in self.headers:
            content_type = self.headers['content-type'].lower()
            match = re.search(r'charset=(\S+)', content_type)
            if match:
                encoding = match.group(1)
        if encoding is None:
            encoding = 'utf-8'  # assume utf-8

        return encoding

    def json(self, check=True):
        if check:
            try:
                self._check_status_code()
            except HTTPError:
                logger.exception("Original content with failed response: %s",
                                 self.content)
                raise
        return json.loads(self.content.decode(self.encoding))
{'fraction_non_alphanumeric': 0.06893640967923466, 'fraction_numerical': 0.007503282686175202, 'mean_word_length': 1.8442251267004535, 'pattern_counts': {'":': 2, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 1, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7504029', 'n_tokens_mistral': 3230, 'n_tokens_neox': 2994, 'n_words': 726}