<commit_before>import six
import json
import platform
import time
import datetime
import types
import re
from six.moves.urllib.parse import urlencode, quote_plus, urlparse

from .version import VERSION

# use urlfetch as request_lib on google app engine, otherwise use requests
request_lib = None
try:
    from google.appengine.api import urlfetch
    request_lib = 'urlfetch'
except ImportError:
    try:
        import requests
        request_lib = 'requests'
    except ImportError:
        raise ImportError('EasyPost requires an up to date requests library. '
                          'Update requests via "pip install -U requests" or '
                          'contact us at <EMAIL>.')

    try:
        version = requests.__version__
        major, minor, patch = [int(i) for i in version.split('.')]
    except Exception:
        raise ImportError('EasyPost requires an up to date requests library. '
                          'Update requests via "pip install -U requests" or contact '
                          'us at <EMAIL>.')
    else:
        if major < 1:
            raise ImportError('EasyPost requires an up to date requests library. Update '
                              'requests via "pip install -U requests" or contact us '
                              'at <EMAIL>.')

# config
api_key = None
api_base = 'https://api.easypost.com/v2'


class Error(Exception):
    def __init__(self, message=None, http_status=None, http_body=None):
        super(Error, self).__init__(message)
        self.http_status = http_status
        self.http_body = http_body
        try:
            self.json_body = json.loads(http_body)
        except:
            self.json_body = None

        self.param = None
        try:
            self.param = self.json_body['error'].get('param', None)
        except:
            pass


def convert_to_easypost_object(response, api_key):
    types = {
        'Address': Address,
        'ScanForm': ScanForm,
        'CustomsItem': CustomsItem,
        'CustomsInfo': CustomsInfo,
        'Parcel': Parcel,
        'Shipment': Shipment,
        'Rate': Rate,
        'Refund': Refund,
        'Batch': Batch,
        'Event': Event,
        'Tracker': Tracker,
        'Pickup': Pickup,
        'Order': Order,
        'PickupRate': PickupRate,
        'PostageLabel': PostageLabel
    }

    prefixes = {
        'adr': Address,
        'sf': ScanForm,
        'evt': Event,
        'cstitem': CustomsItem,
        'cstinfo': CustomsInfo,
        'prcl': Parcel,
        'shp': Shipment,
        'rate': Rate,
        'rfnd': Refund,
        'batch': Batch,
        'trk': Tracker,
        'order': Order,
        'pickup': Pickup,
        'pickuprate': PickupRate,
        'pl': PostageLabel
    }

    if isinstance(response, list):
        return [convert_to_easypost_object(r, api_key) for r in response]
    elif isinstance(response, dict):
        response = response.copy()
        cls_name = response.get('object', EasyPostObject)
        cls_id = response.get('id', None)
        if isinstance(cls_name, six.string_types):
            cls = types.get(cls_name, EasyPostObject)
        elif cls_id is not None:
            cls = prefixes.get(cls_id[0:cls_id.find('_')], EasyPostObject)
        else:
            cls = EasyPostObject
        return cls.construct_from(response, api_key)
    else:
        return response


class Requestor(object):
    def __init__(self, local_api_key=None):
        self.api_key = local_api_key

    @classmethod
    def api_url(cls, url=None):
        url = url or ''
        return '%s%s' % (api_base, url)

    @classmethod
    def _utf8(cls, value):
        if hasattr(value, 'decode'):
            return value.decode('utf-8').encode('utf-8')
        else:
            return value

    @classmethod
    def encode_dict(cls, out, key, dict_value):
        n = {}
        for k, v in six.iteritems(dict_value):
            k = cls._utf8(k)
            v = cls._utf8(v)
            n["%s[%s]" % (key, k)] = v
        out.extend(cls._encode_inner(n))

    @classmethod
    def encode_list(cls, out, key, list_value):
        n = {}
        for k, v in enumerate(list_value):
            v = cls._utf8(v)
            n["%s[%s]" % (key, k)] = v
        out.extend(cls._encode_inner(n))

    @classmethod
    def encode_datetime(cls, out, key, dt_value):
        utc_timestamp = int(time.mktime(dt_value.timetuple()))
        out.append((key, utc_timestamp))

    @classmethod
    def encode_none(cls, out, key, value):
        pass # do not include None-valued params in request

    @classmethod<commit_msg>Fix encoding errors with non-ASCII characters.<commit_after>import six
import json
import platform
import time
import datetime
import types
import re
from six.moves.urllib.parse import urlencode, quote_plus, urlparse

from .version import VERSION

# use urlfetch as request_lib on google app engine, otherwise use requests
request_lib = None
try:
    from google.appengine.api import urlfetch
    request_lib = 'urlfetch'
except ImportError:
    try:
        import requests
        request_lib = 'requests'
    except ImportError:
        raise ImportError('EasyPost requires an up to date requests library. '
                          'Update requests via "pip install -U requests" or '
                          'contact us at <EMAIL>.')

    try:
        version = requests.__version__
        major, minor, patch = [int(i) for i in version.split('.')]
    except Exception:
        raise ImportError('EasyPost requires an up to date requests library. '
                          'Update requests via "pip install -U requests" or contact '
                          'us at <EMAIL>.')
    else:
        if major < 1:
            raise ImportError('EasyPost requires an up to date requests library. Update '
                              'requests via "pip install -U requests" or contact us '
                              'at <EMAIL>.')

# config
api_key = None
api_base = 'https://api.easypost.com/v2'


class Error(Exception):
    def __init__(self, message=None, http_status=None, http_body=None):
        super(Error, self).__init__(message)
        self.http_status = http_status
        self.http_body = http_body
        try:
            self.json_body = json.loads(http_body)
        except:
            self.json_body = None

        self.param = None
        try:
            self.param = self.json_body['error'].get('param', None)
        except:
            pass


def convert_to_easypost_object(response, api_key):
    types = {
        'Address': Address,
        'ScanForm': ScanForm,
        'CustomsItem': CustomsItem,
        'CustomsInfo': CustomsInfo,
        'Parcel': Parcel,
        'Shipment': Shipment,
        'Rate': Rate,
        'Refund': Refund,
        'Batch': Batch,
        'Event': Event,
        'Tracker': Tracker,
        'Pickup': Pickup,
        'Order': Order,
        'PickupRate': PickupRate,
        'PostageLabel': PostageLabel
    }

    prefixes = {
        'adr': Address,
        'sf': ScanForm,
        'evt': Event,
        'cstitem': CustomsItem,
        'cstinfo': CustomsInfo,
        'prcl': Parcel,
        'shp': Shipment,
        'rate': Rate,
        'rfnd': Refund,
        'batch': Batch,
        'trk': Tracker,
        'order': Order,
        'pickup': Pickup,
        'pickuprate': PickupRate,
        'pl': PostageLabel
    }

    if isinstance(response, list):
        return [convert_to_easypost_object(r, api_key) for r in response]
    elif isinstance(response, dict):
        response = response.copy()
        cls_name = response.get('object', EasyPostObject)
        cls_id = response.get('id', None)
        if isinstance(cls_name, six.string_types):
            cls = types.get(cls_name, EasyPostObject)
        elif cls_id is not None:
            cls = prefixes.get(cls_id[0:cls_id.find('_')], EasyPostObject)
        else:
            cls = EasyPostObject
        return cls.construct_from(response, api_key)
    else:
        return response


class Requestor(object):
    def __init__(self, local_api_key=None):
        self.api_key = local_api_key

    @classmethod
    def api_url(cls, url=None):
        url = url or ''
        return '%s%s' % (api_base, url)

    @classmethod
    def _utf8(cls, value):
        if isinstance(value, six.text_type):
            return value.encode('utf-8')
        else:
            return value

    @classmethod
    def encode_dict(cls, out, key, dict_value):
        n = {}
        for k, v in six.iteritems(dict_value):
            k = cls._utf8(k)
            v = cls._utf8(v)
            n["%s[%s]" % (key, k)] = v
        out.extend(cls._encode_inner(n))

    @classmethod
    def encode_list(cls, out, key, list_value):
        n = {}
        for k, v in enumerate(list_value):
            v = cls._utf8(v)
            n["%s[%s]" % (key, k)] = v
        out.extend(cls._encode_inner(n))

    @classmethod
    def encode_datetime(cls, out, key, dt_value):
        utc_timestamp = int(time.mktime(dt_value.timetuple()))
        out.append((key, utc_timestamp))

    @classmethod
    def encode_none(cls, out, key, value):
        pass # do not include None-valued params in request

    @classmethod
{'fraction_non_alphanumeric': 0.1078590785907859, 'fraction_numerical': 0.0018428184281842818, 'mean_word_length': 2.06613492854769, 'pattern_counts': {'":': 0, '<': 11, '<?xml version=': 0, '>': 9, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4501651', 'n_tokens_mistral': 2815, 'n_tokens_neox': 2764, 'n_words': 742}