<commit_before>import json
import logging
import os
from collections import defaultdict
from time import sleep, time

import requests


class ApiCaller:
    class ApiResultNotOk(Exception):
        pass

    def __init__(self, server, test_connection=True, show_api_definition=False, record_destination=None):
        self.server = server
        self.event_offset = 0
        self.record_destination = record_destination
        self.record_indexes = defaultdict(int)
        if test_connection:
            self._print_api_version()
        if show_api_definition:
            logging.debug("API definition:")
            logging.debug(self.api_help_parser())

    def _print_api_version(self):
        r = self._call('version')
        logging.info("API version: {}".format(r))
        for k, v in r.items():
            if v not in self.server.settings.api_compatibility[k]:
                logging.warning("{} is {}, not compatible with known versions {}".format(
                    k,
                    v,
                    self.server.settings.api_compatibility[k])
                )

    def _cleanup_parameters(self, value: str):
        return str(value).translate({ord(i): None for i in '?&#'})

    def _call(self, path: str, params={}):  # TODO handle errors
        url = "{url}/api/{path}?{params}".format(
            url=self.server.api_url,
            path=path,
            params='&'.join(
                [
                    "{k}={v}".format(k=self._cleanup_parameters(k), v=self._cleanup_parameters(v))
                    for k, v in params.items()
                ])
        )
        r = None
        start_time = time()
        while r is None:
            try:
                r = requests.get(url)
            except (requests.ConnectionError, requests.HTTPError) as e:
                if time() - start_time > 60:
                    raise e
                logging.error("Request failed with {}, retrying".format(e))
                sleep(1)
        if not r.ok:
            message = 'Request to {url} returned {code}'.format(url=url, code=r.status_code)
            logging.warning(message)
            raise self.ApiResultNotOk(message)
        parsed = json.loads(r.text)
        if parsed['result'] != 'ok':
            message = 'Request to {url} result was {result}'.format(url=url, result=parsed['result'])
            logging.warning(message)
            raise self.ApiResultNotOk(message)
        return parsed.get('response', None)

    def record_result(self, type, content):
        with open(os.path.join(
                os.getcwd(),
                self.record_destination,
                "{}-{}-{}.json".format(self.record_indexes['total'], type, self.record_indexes[type])
        ), 'w') as f:
            json.dump(content, f, indent=4)
        self.record_indexes[type] += 1
        self.record_indexes['total'] += 1

    def get_lists(self):
        return self._call('list/all')

    def get_status(self, attributes=True, members=True, participants=True):
        params = {
            'attributes': int(attributes),
            'members': int(members),
            'participants': int(participants)
        }
        result = self._call("session/status", params)<commit_msg>Retry all errors<commit_after>import json
import logging
import os
from collections import defaultdict
from time import sleep, time

import requests


class ApiCaller:
    class ApiResultNotOk(Exception):
        pass

    def __init__(self, server, test_connection=True, show_api_definition=False, record_destination=None):
        self.server = server
        self.event_offset = 0
        self.record_destination = record_destination
        self.record_indexes = defaultdict(int)
        if test_connection:
            self._print_api_version()
        if show_api_definition:
            logging.debug("API definition:")
            logging.debug(self.api_help_parser())

    def _print_api_version(self):
        r = self._call('version')
        logging.info("API version: {}".format(r))
        for k, v in r.items():
            if v not in self.server.settings.api_compatibility[k]:
                logging.warning("{} is {}, not compatible with known versions {}".format(
                    k,
                    v,
                    self.server.settings.api_compatibility[k])
                )

    def _cleanup_parameters(self, value: str):
        return str(value).translate({ord(i): None for i in '?&#'})

    def _call(self, path: str, params={}):  # TODO handle errors
        url = "{url}/api/{path}?{params}".format(
            url=self.server.api_url,
            path=path,
            params='&'.join(
                [
                    "{k}={v}".format(k=self._cleanup_parameters(k), v=self._cleanup_parameters(v))
                    for k, v in params.items()
                ])
        )
        r = None
        start_time = time()
        while r is None:
            try:
                r = requests.get(url)
                if not r.ok:
                    message = 'Request to {url} returned {code}'.format(url=url, code=r.status_code)
                    logging.warning(message)
                    raise self.ApiResultNotOk(message)
                parsed = json.loads(r.text)
                if parsed['result'] != 'ok':
                    message = 'Request to {url} result was {result}'.format(url=url, result=parsed['result'])
                    logging.warning(message)
                    raise self.ApiResultNotOk(message)
            except (requests.ConnectionError, requests.HTTPError, self.ApiResultNotOk) as e:
                if time() - start_time > 60:
                    raise e
                logging.error("Request failed with {}, retrying".format(e))
                sleep(1)
        return parsed.get('response', None)

    def record_result(self, type, content):
        with open(os.path.join(
                os.getcwd(),
                self.record_destination,
                "{}-{}-{}.json".format(self.record_indexes['total'], type, self.record_indexes[type])
        ), 'w') as f:
            json.dump(content, f, indent=4)
        self.record_indexes[type] += 1
        self.record_indexes['total'] += 1

    def get_lists(self):
        return self._call('list/all')

    def get_status(self, attributes=True, members=True, participants=True):
        params = {
            'attributes': int(attributes),
            'members': int(members),
            'participants': int(participants)
        }
        result = self._call("session/status", params)
{'fraction_non_alphanumeric': 0.11868533171028606, 'fraction_numerical': 0.002130249543517955, 'mean_word_length': 1.922632281013784, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6510004', 'n_tokens_mistral': 1841, 'n_tokens_neox': 1783, 'n_words': 404}