<commit_before>import asyncio
import requests
import functools

from meteora import utils
import time
import datetime


class Backoff():
    """
    Implements exponential backoff.
    """
    def __init__(self, maxretries=8):
        self.retry = 0
        self.maxretries = maxretries
        self.first = True

    def loop(self):
        if self.first:
            self.first = False
            return True
        else:
            return self.retry < self.maxretries

    def fail(self):
        self.retry += 1
        delay = 2 ** self.retry
        time.sleep(delay)


class Requestor(object):
    """
    Main class to generate requests.
    """
    def __init__(self, url, method=utils.GET, number_of_requests=None,
                 *args, **kwargs):
        """
        """
        self.method = method
        self.url = url
        self.number_of_requests = number_of_requests
        self.results = []
        self.args = args
        self.kwargs = kwargs

    def start_requests(self):
        loop = asyncio.get_event_loop()
        results = loop.run_until_complete(
            self._run_requests(
                self.url, self.number_of_requests,
            )
        )
        self.results = Result(results)

    def _do_request(self, url, num_requests):
        responses = []
        backoff = Backoff()
        for i in range(num_requests):
            while backoff.loop():
                start_time = datetime.datetime.now().microsecond
                if self.method == utils.GET:
                    response = requests.get(url, *self.args, **self.kwargs)
                elif self.method == utils.POST:
                    response = requests.post(url, *self.args, **self.kwargs)
                    # TODO add other methods
                end_time = datetime.datetime.now().microsecond
                response.execution_time = end_time - start_time
                if response.status_code in [402, 403, 408, 503, 504]:
                    print ("Backing off due to status code: %d" % response[i])
                    backoff.fail()
                else:
                    responses.append(response)
                    break

        return responses

    @asyncio.coroutine
    def _run_requests(self, url, num_requests, num_concurrent_users=1):
        loop = asyncio.get_event_loop()
        futures = []
        responses = []<commit_msg>Add the rest of HTTP methods<commit_after>import asyncio
import requests
import functools

from meteora import utils
import time
import datetime


class Backoff():
    """
    Implements exponential backoff.
    """
    def __init__(self, maxretries=8):
        self.retry = 0
        self.maxretries = maxretries
        self.first = True

    def loop(self):
        if self.first:
            self.first = False
            return True
        else:
            return self.retry < self.maxretries

    def fail(self):
        self.retry += 1
        delay = 2 ** self.retry
        time.sleep(delay)


class Requestor(object):
    """
    Main class to generate requests.
    """
    def __init__(self, url, method=utils.GET, number_of_requests=None,
                 *args, **kwargs):
        """
        """
        self.method = method
        self.url = url
        self.number_of_requests = number_of_requests
        self.results = []
        self.args = args
        self.kwargs = kwargs

    def start_requests(self):
        loop = asyncio.get_event_loop()
        results = loop.run_until_complete(
            self._run_requests(
                self.url, self.number_of_requests,
            )
        )
        self.results = Result(results)

    def _do_request(self, url, num_requests):
        responses = []
        backoff = Backoff()
        for i in range(num_requests):
            while backoff.loop():
                start_time = datetime.datetime.now().microsecond
                if self.method == utils.GET:
                    response = requests.get(url, *self.args, **self.kwargs)
                elif self.method == utils.POST:
                    response = requests.post(url, *self.args, **self.kwargs)
                    # TODO add other methods
                elif self.method == utils.PUT:
                    response = request.put(url, *self.args, **self.kwargs)
                elif self.method == utils.HEAD:
                    response = request.head(url, *self.args, **self.kwargs)
                elif self.method == utils.DELETE:
                    response = request.delete(url, *self.args, **self.kwargs)
                elif self.method == utils.PATCH:
                    response = requests.patch(url, *self.args, **self.jwargs)
                else:
                    raise NotImplementedError()
                end_time = datetime.datetime.now().microsecond
                response.execution_time = end_time - start_time
                if response.status_code in [402, 403, 408, 503, 504]:
                    print ("Backing off due to status code: %d" % response[i])
                    backoff.fail()
                else:
                    responses.append(response)
                    break

        return responses

    @asyncio.coroutine
    def _run_requests(self, url, num_requests, num_concurrent_users=1):
        loop = asyncio.get_event_loop()
        futures = []
        responses = []
{'fraction_non_alphanumeric': 0.09592370979805535, 'fraction_numerical': 0.0074794315632011965, 'mean_word_length': 1.6493313521545319, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4292285', 'n_tokens_mistral': 1499, 'n_tokens_neox': 1430, 'n_words': 346}