<commit_before># Tweepy
# Copyright 2009 <NAME>
# See LICENSE

import httplib
import urllib
import time

from . parsers import parse_error
from . error import TweepError

try:
    import json #Python >= 2.6
except ImportError:
    try:
        import simplejson as json #Python < 2.6
    except ImportError:
        try:
            from django.utils import simplejson as json #Google App Engine
        except ImportError:
            raise ImportError, "Can't load a json library"


def bind_api(path, parser, allowed_param=[], method='GET', require_auth=False,
              timeout=None, host=None):

    def _call(api, *args, **kargs):
        # If require auth, throw exception if credentials not provided
        if require_auth and not api.auth_handler:
            raise TweepError('Authentication required!')

        # check for post data
        post_data = kargs.pop('post_data', None)

        # check for retry request parameters
        retry_count = kargs.pop('retry_count', api.retry_count)
        retry_delay = kargs.pop('retry_delay', api.retry_delay)
        retry_errors = kargs.pop('retry_errors', api.retry_errors)

        # check for headers
        headers = kargs.pop('headers', {})

        # build parameter dict
        if allowed_param:
            parameters = {}
            for idx, arg in enumerate(args):
                encoded_arg = arg.encode('utf-8')  # must encode parameters to utf-8
                try:
                    parameters[allowed_param[idx]] = encoded_arg
                except IndexError:
                    raise TweepError('Too many parameters supplied!')
            for k, arg in kargs.items():
                if arg is None:
                    continue
                if k in parameters:
                    raise TweepError('Multiple values for parameter %s supplied!' % k)
                if k not in allowed_param:
                    raise TweepError('Invalid parameter %s supplied!' % k)
                parameters[k] = arg.encode('utf-8')
        else:
            if len(args) > 0 or len(kargs) > 0:
                raise TweepError('This method takes no parameters!')
            parameters = None

        # Build url with parameters
        if parameters:
            url = '%s?%s' % (api.api_root + path, urllib.urlencode(parameters))
        else:
            url = api.api_root + path

        # Check cache if caching enabled and method is GET
        if api.cache and method == 'GET':
            cache_result = api.cache.get(url, timeout)
            # if cache result found and not expired, return it
            if cache_result:
                # must restore api reference
                if isinstance(cache_result, list):
                    for result in cache_result:
                        result._api = api
                else:
                    cache_result._api = api
                return cache_result

        # get scheme and host
        if api.secure:
            scheme = 'https://'
        else:
            scheme = 'http://'
        _host = host or api.host
<commit_msg>Only utf-8 encode unicode parameters in binder.<commit_after># Tweepy
# Copyright 2009 <NAME>
# See LICENSE

import httplib
import urllib
import time

from . parsers import parse_error
from . error import TweepError

try:
    import json #Python >= 2.6
except ImportError:
    try:
        import simplejson as json #Python < 2.6
    except ImportError:
        try:
            from django.utils import simplejson as json #Google App Engine
        except ImportError:
            raise ImportError, "Can't load a json library"


def bind_api(path, parser, allowed_param=[], method='GET', require_auth=False,
              timeout=None, host=None):

    def _call(api, *args, **kargs):
        # If require auth, throw exception if credentials not provided
        if require_auth and not api.auth_handler:
            raise TweepError('Authentication required!')

        # check for post data
        post_data = kargs.pop('post_data', None)

        # check for retry request parameters
        retry_count = kargs.pop('retry_count', api.retry_count)
        retry_delay = kargs.pop('retry_delay', api.retry_delay)
        retry_errors = kargs.pop('retry_errors', api.retry_errors)

        # check for headers
        headers = kargs.pop('headers', {})

        # build parameter dict
        if allowed_param:
            parameters = {}
            for idx, arg in enumerate(args):
                if isinstance(arg, unicode):
                    arg = arg.encode('utf-8')
                try:
                    parameters[allowed_param[idx]] = arg
                except IndexError:
                    raise TweepError('Too many parameters supplied!')
            for k, arg in kargs.items():
                if arg is None:
                    continue
                if k in parameters:
                    raise TweepError('Multiple values for parameter %s supplied!' % k)
                if k not in allowed_param:
                    raise TweepError('Invalid parameter %s supplied!' % k)
                if isinstance(arg, unicode):
                    arg = arg.encode('utf-8')
                parameters[k] = arg
        else:
            if len(args) > 0 or len(kargs) > 0:
                raise TweepError('This method takes no parameters!')
            parameters = None

        # Build url with parameters
        if parameters:
            url = '%s?%s' % (api.api_root + path, urllib.urlencode(parameters))
        else:
            url = api.api_root + path

        # Check cache if caching enabled and method is GET
        if api.cache and method == 'GET':
            cache_result = api.cache.get(url, timeout)
            # if cache result found and not expired, return it
            if cache_result:
                # must restore api reference
                if isinstance(cache_result, list):
                    for result in cache_result:
                        result._api = api
                else:
                    cache_result._api = api
                return cache_result

        # get scheme and host
        if api.secure:
            scheme = 'https://'
        else:
            scheme = 'http://'
        _host = host or api.host

{'fraction_non_alphanumeric': 0.08058082016913994, 'fraction_numerical': 0.00414871549385671, 'mean_word_length': 1.7087294727744167, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 11, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6464170', 'n_tokens_mistral': 1668, 'n_tokens_neox': 1592, 'n_words': 539}