<commit_before># coding=UTF-8

"""Provides FCO API Client and decorator."""

from functools import wraps

from fcoclient.exceptions import (NonRecoverableError, RecoverableError)

import requests
import json
from requests import codes as rsc
from time import sleep


# REST Client default settings
REST_RETRY_COUNT = 5
REST_RETRY_DELAY = 30
REST_FAILURE_EXCEPTION = NonRecoverableError
REST_INTERNAL_RETRY = True

REST_API_VERSION = '5.0'
REST_HEADERS = {'content-type': 'application/json'}


# Configurable properties dict keys
PROP_CLIENT_CONFIG = 'auth'
PROP_CLIENT_TOKEN = 'token'
PROP_CLIENT_USERNAME = 'username'
PROP_CLIENT_PASSWORD = 'password'
PROP_CLIENT_API_USERNAME = 'api_user_uuid'
PROP_CLIENT_API_PASSWORD = '<PASSWORD>'
PROP_CLIENT_CUSTOMER = 'customer'
PROP_CLIENT_SERVICE_URL = 'url'
PROP_CLIENT_CA_CERT = 'verify_ca_cert'

# Configurable kwargs keys for client
KW_PAYLOAD = 'payload'
KW_PATTERN = 'pattern'


def _rest_client_retry_and_auth(f):
    """
    Inject authentication and retry on REST failure.

    :param f: function to wrap
    :return: wrapper around function
    """
    @wraps(f)
    def wrapper(self, endpoint, data=None, **kwargs):
        """
        Make an API call to a given endpoint.

        :param endpoint: URL of the endpoint
        :param data: data to include with the request
        :param kwargs: alternative method of including data
        :return: content of a successful response
        """
        # TODO: remove legacy block
        try:
            endpoint = endpoint.format(**kwargs.get('pattern', {}))
        except IndexError:
            NonRecoverableError('Unable to format endpoint, pattern: {}, '
                                'data: {}.' .format(endpoint,
                                                    kwargs.get(KW_PATTERN)))

        url = '{service_url}/rest/user/{api_version}/{endpoint}'.format(
            service_url=self.service_url, api_version=REST_API_VERSION,
            endpoint=endpoint)
        retry_count = self.retry_count
        payload = kwargs.get(KW_PAYLOAD)

        if data:
            payload = data

        while retry_count:
            terminate = False

            self.logger.info('Client function: %s', f.__name__)
            self.logger.info('Client URL: %s', url)
            self.logger.info('Client data: %s', data)
            r = f(self, url, payload, self.auth, self.headers,
                  self.verify)

            self.logger.debug('Client final URL: {}'.format(r.url))
            if len(r.content) > 60:
                self.logger.info('Content: {}'.format(r.content[:57] + '...'))
                self.logger.debug('Full content: {}'.format(r.content))
            else:
                self.logger.info('Content: {}'.format(r.content))
            self.logger.info('Status code: {}'.format(r.status_code))

            if r.status_code == rsc.accepted or r.status_code == rsc.ok:
                self.logger.debug('=' * 60)

                # TODO: convert everything to unicode?
                content = json.loads(r.content)

                def to_str(uni):
                    """Recursively turn unciode to str."""
                    if isinstance(uni, list):
                        gen = enumerate(uni)
                        string = [None]*len(uni)
                    elif isinstance(uni, dict):
                        gen = uni.items()
                        string = {}
                    elif isinstance(uni, basestring):
                        return str(uni)
                    else:
                        return uni
                    for k, v in gen:
                        string[to_str(k)] = to_str(v)
                    return string

                return to_str(content)

            if r.status_code == rsc.too_many_requests:<commit_msg>unspam log with request information<commit_after># coding=UTF-8

"""Provides FCO API Client and decorator."""

from functools import wraps

from fcoclient.exceptions import (NonRecoverableError, RecoverableError)

import requests
import json
from requests import codes as rsc
from time import sleep


# REST Client default settings
REST_RETRY_COUNT = 5
REST_RETRY_DELAY = 30
REST_FAILURE_EXCEPTION = NonRecoverableError
REST_INTERNAL_RETRY = True

REST_API_VERSION = '5.0'
REST_HEADERS = {'content-type': 'application/json'}


# Configurable properties dict keys
PROP_CLIENT_CONFIG = 'auth'
PROP_CLIENT_TOKEN = 'token'
PROP_CLIENT_USERNAME = 'username'
PROP_CLIENT_PASSWORD = 'password'
PROP_CLIENT_API_USERNAME = 'api_user_uuid'
PROP_CLIENT_API_PASSWORD = '<PASSWORD>'
PROP_CLIENT_CUSTOMER = 'customer'
PROP_CLIENT_SERVICE_URL = 'url'
PROP_CLIENT_CA_CERT = 'verify_ca_cert'

# Configurable kwargs keys for client
KW_PAYLOAD = 'payload'
KW_PATTERN = 'pattern'


def _rest_client_retry_and_auth(f):
    """
    Inject authentication and retry on REST failure.

    :param f: function to wrap
    :return: wrapper around function
    """
    @wraps(f)
    def wrapper(self, endpoint, data=None, **kwargs):
        """
        Make an API call to a given endpoint.

        :param endpoint: URL of the endpoint
        :param data: data to include with the request
        :param kwargs: alternative method of including data
        :return: content of a successful response
        """
        # TODO: remove legacy block
        try:
            endpoint = endpoint.format(**kwargs.get('pattern', {}))
        except IndexError:
            NonRecoverableError('Unable to format endpoint, pattern: {}, '
                                'data: {}.' .format(endpoint,
                                                    kwargs.get(KW_PATTERN)))

        url = '{service_url}/rest/user/{api_version}/{endpoint}'.format(
            service_url=self.service_url, api_version=REST_API_VERSION,
            endpoint=endpoint)
        retry_count = self.retry_count
        payload = kwargs.get(KW_PAYLOAD)

        if data:
            payload = data

        while retry_count:
            terminate = False

            self.logger.info('%s %s', f,__name__.capitalize, url)
            self.logger.debug('Client function: %s', f.__name__)
            self.logger.debug('Client URL: %s', url)
            self.logger.debug('Client data: %s', data)
            r = f(self, url, payload, self.auth, self.headers,
                  self.verify)

            self.logger.debug('Client final URL: {}'.format(r.url))
            self.logger.debug('Full content: {}'.format(r.content))
            self.logger.debug('Status code: {}'.format(r.status_code))

            if r.status_code == rsc.accepted or r.status_code == rsc.ok:
                self.logger.debug('=' * 60)

                # TODO: convert everything to unicode?
                content = json.loads(r.content)

                def to_str(uni):
                    """Recursively turn unciode to str."""
                    if isinstance(uni, list):
                        gen = enumerate(uni)
                        string = [None]*len(uni)
                    elif isinstance(uni, dict):
                        gen = uni.items()
                        string = {}
                    elif isinstance(uni, basestring):
                        return str(uni)
                    else:
                        return uni
                    for k, v in gen:
                        string[to_str(k)] = to_str(v)
                    return string

                return to_str(content)

            if r.status_code == rsc.too_many_requests:
{'fraction_non_alphanumeric': 0.09918785780854747, 'fraction_numerical': 0.0026627612834509385, 'mean_word_length': 1.991636798088411, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 6, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '470304', 'n_tokens_mistral': 2153, 'n_tokens_neox': 2067, 'n_words': 531}