<commit_before>"""For binding API calls to python functions."""
from __future__ import print_function

import re
import requests
import time

from six.moves.urllib.parse import quote

from ringplus.utils import convert_to_utf8_str
from ringplus.error import RingPlusError, RateLimitError
from ringplus.error import is_rate_limit_error_message
from ringplus.models import Model

re_path_template = re.compile('{\w+}')


def bind_api(**config):

    class APIMethod(object):

        api = config['api']
        path = config['path']
        payload_type = config.get('payload_type', None)
        payload_list = config.get('payload_list', False)
        allowed_param = config.get('allowed_param', [])
        method = config.get('method', 'GET')
        # require_auth = config.get('require_auth', False)
        # search_api = config.get('search_api', False)
        # upload_api = config.get('upload_api', False)
        use_cache = config.get('use_cache', True)
        session = requests.Session()

        def __init__(self, args, kwargs):
            api = self.api

            self.post_data = kwargs.pop('post_data', None)
            self.retry_count = kwargs.pop('retry_count', api.retry_count)

            self.retry_delay = kwargs.pop('retry_delay', api.retry_delay)
            self.retry_errors = kwargs.pop('retry_errors', api.retry_errors)
            self.wait_on_rate_limit = kwargs.pop('wait_on_rate_limit',
                                                 api.wait_on_rate_limit)
            self.wait_on_rate_limit_notify = kwargs.pop(
                'wait_on_rate_limit_notify', api.wait_on_rate_limit_notify)

            self.parser = kwargs.pop('parser', api.parser)
            self.session.headers = kwargs.pop('headers', {})
            self.build_parameters(args, kwargs)

            # Perform any path variable substitution
            self.build_path()

            self.host = api.host

            # Manually set Host header
            self.session.headers['Host'] = self.host
            # Monitoring rate limits
            self._remaining_calls = None
            self._reset_time = None

        def build_parameters(self, args, kwargs):
            self.session.params = {}
            for idx, arg in enumerate(args):
                if arg is None:
                    continue
                try:
                    utf8str = convert_to_utf8_str(arg)
                    self.session.params[self.allowed_param[idx]] = utf8str
                except IndexError:
                    raise RingPlusError('Too many parameters supplied!')

            for k, arg in kwargs.items():
                if arg is None:
                    continue
                if k in self.session.params:
                    err_str = 'Multiple values for parameter %s supplied!' % k
                    raise RingPlusError(err_str)

                self.session.params[k] = convert_to_utf8_str(arg)

        def build_path(self):
            for variable in re_path_template.findall(self.path):
                name = variable.strip('{}')

                if name == 'account_id' and \
                           'account_id' not in self.session.params and \
                           self.api.auth:
                    value = self.api.auth.get_account_id()
                elif name == 'user_id' and \
                             'user_id' not in self.session.params and \
                             self.api.auth:
                    value = self.api.get_user_id()
                else:
                    try:
                        value = quote(self.session.params[name])
                    except KeyError:
                        raise RingPlusError('No parameter value found for '
                                            'path variable: %s' % name)
                        del self.session.params[name]

                self.path = self.path.replace(variable, value)

        def execute(self):
            self.api.cached_result = False

            # Build the request URL
            url = self.path
            full_url = 'https://' + self.host + url

            # Query the cache if on is available
            # and this use a GET method.
            if self.use_cache and self.api.cache and self.method == 'GET':
                cache_result = self.api.cache.get(url)
                # if cache result found and not expired, return it
                if cache_result:
                    # must restore api reference
                    if isinstance(cache_result, list):
                        for result in cache_result:
                            if isinstance(result, Model):
                                result._api = self.api
                    else:
                        if isinstance(cache_result, Model):
                            cache_result._api = self.api
                    self.api.cached_result = True
                    return cache_result

            # Continue attempting request until successful
            # or maximum number of retries is reached.
            retries_performed = 0
            while retries_performed < self.retry_count + 1:
                # handle running out of api calls
                if self.wait_on_rate_limit:
                    if self._reset_time is not None:
                        if self._remaining_calls is not None:
                            if self._remaining_calls < 1:
                                sleeptime = self._reset_time - int(time.time())
                                if sleeptime > 0:
                                    if self.wait_on_rate_limit_notify:
                                        print("Rate limit reached."\
                                              "Sleeping for:", sleeptime)
                                    time.sleep(sleeptime + 5)

                # Apply authentication
                if self.api.auth:
                    auth = self.api.auth.apply_auth()

                # # Request compression if configured
                # if self.api.compression:
                #     self.session.headers['Accept-encoding'] = 'gzip'

                # Execute request
                try:
                    resp = self.session.request(self.method,
                                                full_url,
                                                data=self.post_data,
                                                timeout=self.api.timeout,
                                                auth=auth,
                                                proxies=self.api.proxy)
                except Exception as e:
                    raise RingPlusError('Failed to send request: %s' % e)

                rem_calls = resp.headers.get('x-rate-limit-remaining')
                if rem_calls is not None:
                        self._remaining_calls = int(rem_calls)
                elif isinstance(self._remaining_calls, int):
                    self._remaining_calls -= 1
                reset_time = resp.headers.get('x-rate-limit-reset')
                if reset_time is not None:
                    self._reset_time = int(reset_time)
                if self.wait_on_rate_limit and self._remaining_calls == 0 and (
                        # if ran out of calls before waiting switching retry last call
                        resp.status_code == 429 or resp.status_code == 420):
                    continue
                retry_delay = self.retry_delay
                # Exit request loop if non-retry error code
                if resp.status_code == 200:
                    break
                elif (resp.status_code == 429 or resp.status_code == 420) and \
                      self.wait_on_rate_limit:
                    if 'retry-after' in resp.headers:
                        retry_delay = float(resp.headers['retry-after'])
                elif self.retry_errors and \
                        resp.status_code not in self.retry_errors:
                    break

                # Sleep before retrying request again
                time.sleep(retry_delay)
                retries_performed += 1

            # If an error was returned, throw an exception
            self.api.last_response = resp
            if resp.status_code and not 200 <= resp.status_code < 300:
                try:
                    error_msg, api_error_code = \
                        self.parser.parse_error(resp.text)
                except Exception:
                    error_msg = "Twitter error response: status code = %s" \
                        % resp.status_code
                    api_error_code = None

                if is_rate_limit_error_message(error_msg):
                    raise RateLimitError(error_msg, resp)
                else:
                    raise RingPlusError(error_msg, resp,
                                        api_code=api_error_code)

            # Parse the response payload
            result = self.parser.parse(self, resp.text)

            # Store result into cache if one is available.
            if self.use_cache and self.api.cache and \
                    self.method == 'GET' and result:
                self.api.cache.store(url, result)

            return result

    def _call(*args, **kwargs):
        method = APIMethod(args, kwargs)
        if kwargs.get('create'):
            return method
        else:
            return method.execute()

    # Set pagination mode
    if 'cursor' in APIMethod.allowed_param:
        _call.pagination_mode = 'cursor'
    elif 'max_id' in APIMethod.allowed_param:
        if 'since_id' in APIMethod.allowed_param:
            _call.pagination_mode = 'id'
    elif 'page' in APIMethod.allowed_param:
        _call.pagination_mode = 'page'

    return _call
<commit_msg>Added version header to bind_api<commit_after>"""For binding API calls to python functions."""
from __future__ import print_function

import re
import requests
import time

from six.moves.urllib.parse import quote

from ringplus.utils import convert_to_utf8_str
from ringplus.error import RingPlusError, RateLimitError
from ringplus.error import is_rate_limit_error_message
from ringplus.models import Model

re_path_template = re.compile('{\w+}')


def bind_api(**config):

    class APIMethod(object):

        api = config['api']
        path = config['path']
        payload_type = config.get('payload_type', None)
        payload_list = config.get('payload_list', False)
        allowed_param = config.get('allowed_param', [])
        method = config.get('method', 'GET')
        # require_auth = config.get('require_auth', False)
        # search_api = config.get('search_api', False)
        # upload_api = config.get('upload_api', False)
        use_cache = config.get('use_cache', True)
        session = requests.Session()

        def __init__(self, args, kwargs):
            api = self.api

            self.post_data = kwargs.pop('post_data', None)
            self.retry_count = kwargs.pop('retry_count', api.retry_count)

            self.retry_delay = kwargs.pop('retry_delay', api.retry_delay)
            self.retry_errors = kwargs.pop('retry_errors', api.retry_errors)
            self.wait_on_rate_limit = kwargs.pop('wait_on_rate_limit',
                                                 api.wait_on_rate_limit)
            self.wait_on_rate_limit_notify = kwargs.pop(
                'wait_on_rate_limit_notify', api.wait_on_rate_limit_notify)

            self.parser = kwargs.pop('parser', api.parser)
            self.session.headers = kwargs.pop('headers', {})
            self.build_parameters(args, kwargs)

            # Perform any path variable substitution
            self.build_path()

            self.host = api.host

            # Manually set Host header
            self.session.headers['Host'] = self.host
            # Set version header
            self.session.headers['Accept'] = 'application/vnd.ringplus.v{}'.\
                format(self.api.version)
            # Monitoring rate limits
            self._remaining_calls = None
            self._reset_time = None

        def build_parameters(self, args, kwargs):
            self.session.params = {}
            for idx, arg in enumerate(args):
                if arg is None:
                    continue
                try:
                    utf8str = convert_to_utf8_str(arg)
                    self.session.params[self.allowed_param[idx]] = utf8str
                except IndexError:
                    raise RingPlusError('Too many parameters supplied!')

            for k, arg in kwargs.items():
                if arg is None:
                    continue
                if k in self.session.params:
                    err_str = 'Multiple values for parameter %s supplied!' % k
                    raise RingPlusError(err_str)

                self.session.params[k] = convert_to_utf8_str(arg)

        def build_path(self):
            for variable in re_path_template.findall(self.path):
                name = variable.strip('{}')

                if name == 'account_id' and \
                           'account_id' not in self.session.params and \
                           self.api.auth:
                    value = self.api.auth.get_account_id()
                elif name == 'user_id' and \
                             'user_id' not in self.session.params and \
                             self.api.auth:
                    value = self.api.get_user_id()
                else:
                    try:
                        value = quote(self.session.params[name])
                    except KeyError:
                        raise RingPlusError('No parameter value found for '
                                            'path variable: %s' % name)
                        del self.session.params[name]

                self.path = self.path.replace(variable, value)

        def execute(self):
            self.api.cached_result = False

            # Build the request URL
            url = self.path
            full_url = 'https://' + self.host + url

            # Query the cache if on is available
            # and this use a GET method.
            if self.use_cache and self.api.cache and self.method == 'GET':
                cache_result = self.api.cache.get(url)
                # if cache result found and not expired, return it
                if cache_result:
                    # must restore api reference
                    if isinstance(cache_result, list):
                        for result in cache_result:
                            if isinstance(result, Model):
                                result._api = self.api
                    else:
                        if isinstance(cache_result, Model):
                            cache_result._api = self.api
                    self.api.cached_result = True
                    return cache_result

            # Continue attempting request until successful
            # or maximum number of retries is reached.
            retries_performed = 0
            while retries_performed < self.retry_count + 1:
                # handle running out of api calls
                if self.wait_on_rate_limit:
                    if self._reset_time is not None:
                        if self._remaining_calls is not None:
                            if self._remaining_calls < 1:
                                sleeptime = self._reset_time - int(time.time())
                                if sleeptime > 0:
                                    if self.wait_on_rate_limit_notify:
                                        print("Rate limit reached."\
                                              "Sleeping for:", sleeptime)
                                    time.sleep(sleeptime + 5)

                # Apply authentication
                if self.api.auth:
                    auth = self.api.auth.apply_auth()

                # # Request compression if configured
                # if self.api.compression:
                #     self.session.headers['Accept-encoding'] = 'gzip'

                # Execute request
                try:
                    resp = self.session.request(self.method,
                                                full_url,
                                                data=self.post_data,
                                                timeout=self.api.timeout,
                                                auth=auth,
                                                proxies=self.api.proxy)
                except Exception as e:
                    raise RingPlusError('Failed to send request: %s' % e)

                rem_calls = resp.headers.get('x-rate-limit-remaining')
                if rem_calls is not None:
                        self._remaining_calls = int(rem_calls)
                elif isinstance(self._remaining_calls, int):
                    self._remaining_calls -= 1
                reset_time = resp.headers.get('x-rate-limit-reset')
                if reset_time is not None:
                    self._reset_time = int(reset_time)
                if self.wait_on_rate_limit and self._remaining_calls == 0 and (
                        # if ran out of calls before waiting switching retry last call
                        resp.status_code == 429 or resp.status_code == 420):
                    continue
                retry_delay = self.retry_delay
                # Exit request loop if non-retry error code
                if resp.status_code == 200:
                    break
                elif (resp.status_code == 429 or resp.status_code == 420) and \
                      self.wait_on_rate_limit:
                    if 'retry-after' in resp.headers:
                        retry_delay = float(resp.headers['retry-after'])
                elif self.retry_errors and \
                        resp.status_code not in self.retry_errors:
                    break

                # Sleep before retrying request again
                time.sleep(retry_delay)
                retries_performed += 1

            # If an error was returned, throw an exception
            self.api.last_response = resp
            if resp.status_code and not 200 <= resp.status_code < 300:
                try:
                    error_msg, api_error_code = \
                        self.parser.parse_error(resp.text)
                except Exception:
                    error_msg = "Twitter error response: status code = %s" \
                        % resp.status_code
                    api_error_code = None

                if is_rate_limit_error_message(error_msg):
                    raise RateLimitError(error_msg, resp)
                else:
                    raise RingPlusError(error_msg, resp,
                                        api_code=api_error_code)

            # Parse the response payload
            result = self.parser.parse(self, resp.text)

            # Store result into cache if one is available.
            if self.use_cache and self.api.cache and \
                    self.method == 'GET' and result:
                self.api.cache.store(url, result)

            return result

    def _call(*args, **kwargs):
        method = APIMethod(args, kwargs)
        if kwargs.get('create'):
            return method
        else:
            return method.execute()

    # Set pagination mode
    if 'cursor' in APIMethod.allowed_param:
        _call.pagination_mode = 'cursor'
    elif 'max_id' in APIMethod.allowed_param:
        if 'since_id' in APIMethod.allowed_param:
            _call.pagination_mode = 'id'
    elif 'page' in APIMethod.allowed_param:
        _call.pagination_mode = 'page'

    return _call

{'fraction_non_alphanumeric': 0.07606207071991859, 'fraction_numerical': 0.0034596794708725517, 'mean_word_length': 1.3239536533459446, 'pattern_counts': {'":': 0, '<': 11, '<?xml version=': 0, '>': 5, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 42, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3697445', 'n_tokens_mistral': 4962, 'n_tokens_neox': 4719, 'n_words': 1200}