<commit_before># Copyright (c) 2011 OpenStack, LLC.
# All Rights Reserved.
#
#    Licensed under the Apache License, Version 2.0 (the "License"); you may
#    not use this file except in compliance with the License. You may obtain
#    a copy of the License at
#
#         http://www.apache.org/licenses/LICENSE-2.0
#
#    Unless required by applicable law or agreed to in writing, software
#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
#    License for the specific language governing permissions and limitations
#    under the License.

import httplib2
import logging
import os
import time
import urlparse

try:
    import json
except ImportError:
    import simplejson as json

# Python 2.5 compat fix
if not hasattr(urlparse, 'parse_qsl'):
    import cgi
    urlparse.parse_qsl = cgi.parse_qsl

from reddwarfclient import auth
from reddwarfclient import exceptions


_logger = logging.getLogger(__name__)
if 'REDDWARFCLIENT_DEBUG' in os.environ and os.environ['REDDWARFCLIENT_DEBUG']:
    ch = logging.StreamHandler()
    _logger.setLevel(logging.DEBUG)
    _logger.addHandler(ch)


class ReddwarfHTTPClient(httplib2.Http):

    USER_AGENT = 'python-reddwarfclient'

    def __init__(self, user, password, tenant, auth_url, service_name,
                 service_url=None,
                 auth_strategy=None, insecure=False,
                 timeout=None, proxy_tenant_id=None,
                 proxy_token=None, region_name=None,
                 endpoint_type='publicURL', service_type=None,
                 timings=False):

        super(ReddwarfHTTPClient, self).__init__(timeout=timeout)

        self.username = user
        self.password = password
        self.tenant = tenant
        self.auth_url = auth_url.rstrip('/')
        self.region_name = region_name
        self.endpoint_type = endpoint_type
        self.service_url = service_url
        self.service_type = service_type
        self.service_name = service_name
        self.timings = timings

        self.times = []  # [("item", starttime, endtime), ...]

        self.auth_token = None
        self.proxy_token = proxy_token
        self.proxy_tenant_id = proxy_tenant_id

        # httplib2 overrides
        self.force_exception_to_status_code = True
        self.disable_ssl_certificate_validation = insecure
        self.authenticator = auth.Authenticator(self, auth_strategy,
                                                self.auth_url, self.username,
                                                self.password, self.tenant,
                                                region=region_name,
                                                service_type=service_type,
                                                service_name=service_name,
                                                service_url=service_url)

    def get_timings(self):
        return self.times

    def http_log(self, args, kwargs, resp, body):
        if not _logger.isEnabledFor(logging.DEBUG):
            return

        string_parts = ['curl -i']
        for element in args:
            if element in ('GET', 'POST'):
                string_parts.append(' -X %s' % element)
            else:
                string_parts.append(' %s' % element)

        for element in kwargs['headers']:
            header = ' -H "%s: %s"' % (element, kwargs['headers'][element])
            string_parts.append(header)

        _logger.debug("REQ: %s\n" % "".join(string_parts))
        if 'body' in kwargs:
            _logger.debug("REQ BODY: %s\n" % (kwargs['body']))
        _logger.debug("RESP:%s %s\n", resp, body)

    def request(self, *args, **kwargs):
        kwargs.setdefault('headers', kwargs.get('headers', {}))
        kwargs['headers']['User-Agent'] = self.USER_AGENT
        kwargs['headers']['Accept'] = 'application/json'
        if 'body' in kwargs:
            kwargs['headers']['Content-Type'] = 'application/json'
            kwargs['body'] = json.dumps(kwargs['body'])

        resp, body = super(ReddwarfHTTPClient, self).request(*args, **kwargs)

        # Save this in case anyone wants it.
        self.last_response = (resp, body)
        self.http_log(args, kwargs, resp, body)

        if body:
            try:
                body = json.loads(body)
            except ValueError:<commit_msg>Added a "pretty_print" option which makes captured output look even better.<commit_after># Copyright (c) 2011 OpenStack, LLC.
# All Rights Reserved.
#
#    Licensed under the Apache License, Version 2.0 (the "License"); you may
#    not use this file except in compliance with the License. You may obtain
#    a copy of the License at
#
#         http://www.apache.org/licenses/LICENSE-2.0
#
#    Unless required by applicable law or agreed to in writing, software
#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
#    License for the specific language governing permissions and limitations
#    under the License.

import httplib2
import logging
import os
import time
import urlparse

try:
    import json
except ImportError:
    import simplejson as json

# Python 2.5 compat fix
if not hasattr(urlparse, 'parse_qsl'):
    import cgi
    urlparse.parse_qsl = cgi.parse_qsl

from reddwarfclient import auth
from reddwarfclient import exceptions


_logger = logging.getLogger(__name__)
RDC_PP = os.environ.get("RDC_PP", "False") == "True"


if 'REDDWARFCLIENT_DEBUG' in os.environ and os.environ['REDDWARFCLIENT_DEBUG']:
    ch = logging.StreamHandler()
    _logger.setLevel(logging.DEBUG)
    _logger.addHandler(ch)


class ReddwarfHTTPClient(httplib2.Http):

    USER_AGENT = 'python-reddwarfclient'

    def __init__(self, user, password, tenant, auth_url, service_name,
                 service_url=None,
                 auth_strategy=None, insecure=False,
                 timeout=None, proxy_tenant_id=None,
                 proxy_token=None, region_name=None,
                 endpoint_type='publicURL', service_type=None,
                 timings=False):

        super(ReddwarfHTTPClient, self).__init__(timeout=timeout)

        self.username = user
        self.password = password
        self.tenant = tenant
        self.auth_url = auth_url.rstrip('/')
        self.region_name = region_name
        self.endpoint_type = endpoint_type
        self.service_url = service_url
        self.service_type = service_type
        self.service_name = service_name
        self.timings = timings

        self.times = []  # [("item", starttime, endtime), ...]

        self.auth_token = None
        self.proxy_token = proxy_token
        self.proxy_tenant_id = proxy_tenant_id

        # httplib2 overrides
        self.force_exception_to_status_code = True
        self.disable_ssl_certificate_validation = insecure
        self.authenticator = auth.Authenticator(self, auth_strategy,
                                                self.auth_url, self.username,
                                                self.password, self.tenant,
                                                region=region_name,
                                                service_type=service_type,
                                                service_name=service_name,
                                                service_url=service_url)

    def get_timings(self):
        return self.times

    def http_log(self, args, kwargs, resp, body):
        if not RDC_PP:
            self.simple_log(args, kwargs, resp, body)
        else:
            self.pretty_log(args, kwargs, resp, body)

    def simple_log(self, args, kwargs, resp, body):
        if not _logger.isEnabledFor(logging.DEBUG):
            return

        string_parts = ['curl -i']
        for element in args:
            if element in ('GET', 'POST'):
                string_parts.append(' -X %s' % element)
            else:
                string_parts.append(' %s' % element)

        for element in kwargs['headers']:
            header = ' -H "%s: %s"' % (element, kwargs['headers'][element])
            string_parts.append(header)

        _logger.debug("REQ: %s\n" % "".join(string_parts))
        if 'body' in kwargs:
            _logger.debug("REQ BODY: %s\n" % (kwargs['body']))
        _logger.debug("RESP:%s %s\n", resp, body)

    def pretty_log(self, args, kwargs, resp, body):
        from reddwarfclient import common
        if not _logger.isEnabledFor(logging.DEBUG):
            return

        string_parts = ['curl -i']
        for element in args:
            if element in ('GET', 'POST'):
                string_parts.append(' -X %s' % element)
            else:
                string_parts.append(' %s' % element)

        for element in kwargs['headers']:
            header = ' -H "%s: %s"' % (element, kwargs['headers'][element])
            string_parts.append(header)

        curl_cmd = "".join(string_parts)
        _logger.debug("REQUEST:")
        if 'body' in kwargs:
            _logger.debug("%s -d '%s'" % (curl_cmd, kwargs['body']))
            try:
                req_body = json.dumps(json.loads(kwargs['body']),
                                      sort_keys=True, indent=4)
            except:
                req_body = kwargs['body']
            _logger.debug("BODY: %s\n" % (req_body))
        else:
            _logger.debug(curl_cmd)

        try:
            resp_body = json.dumps(json.loads(body), sort_keys=True, indent=4)
        except:
            resp_body = body
        _logger.debug("RESPONSE HEADERS: %s" % resp)
        _logger.debug("RESPONSE BODY   : %s" % resp_body)

    def request(self, *args, **kwargs):
        kwargs.setdefault('headers', kwargs.get('headers', {}))
        kwargs['headers']['User-Agent'] = self.USER_AGENT
        kwargs['headers']['Accept'] = 'application/json'
        if 'body' in kwargs:
            kwargs['headers']['Content-Type'] = 'application/json'
            kwargs['body'] = json.dumps(kwargs['body'])

        resp, body = super(ReddwarfHTTPClient, self).request(*args, **kwargs)

        # Save this in case anyone wants it.
        self.last_response = (resp, body)
        self.http_log(args, kwargs, resp, body)

        if body:
            try:
                body = json.loads(body)
            except ValueError:
{'fraction_non_alphanumeric': 0.10196229318968834, 'fraction_numerical': 0.0026933435936898806, 'mean_word_length': 2.0824192113845243, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 25, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7116122', 'n_tokens_mistral': 2906, 'n_tokens_neox': 2872, 'n_words': 741}