<commit_before>#/usr/bin/env python
# -*- coding: utf-8 -*-

from gevent import monkey
monkey.patch_all()

import time
import requests
import argparse
import gevent
import os
import sys

from gevent.pool import Pool
from collections import defaultdict
from sys import exit


HTTP_VERBS = 'GET', 'POST', 'PUT', 'DELETE'
_stats = defaultdict(list)


def progress(func):
    """
    @decorator:
        display an tiret for each query treatment
    """
    def wrapper(*args):
        sys.stdout.write('-')
        sys.stdout.flush()
        return func(*args)
    return wrapper


def cookies_parse(cookies):
    """
    @args:
        <list_of_cookies>

    @return:
        <dict_of_cookies>
    """
    _cookies = {}
    try:
        for cookie in cookies:
            _cookie = cookie.split(':')
            if len(_cookie) != 2:
                raise Exception
            else:
                _cookie = iter(reversed(_cookie))
                _cookies[_cookie.next()] = _cookie.next()
    except Exception:
        sys.stdout.write('discarding invalid cookies: {}\n'.format(cookie))
        exit(1)
    else:
        return _cookies


def clear_stats():
    """
    keeps consistent stats when --repeat is used
    """
    global _stats
    del _stats
    _stats = defaultdict(list)


# TODO: fix bug AttributeError: 'Greenlet' object has no attribute '_run'
@progress
def call(method, url, options):
    """
    @args:
        <method_of_request>
        <str_url>
        <dict_requests_options>

    made a request and updates the stats
    """
    try:
        start = time.time()
        res = method(url, **options)
        code = res.status_code
    except requests.exceptions.Timeout:
        code = 408
    except Exception:
        code = 404
    finally:
        _stats[code].append(time.time()-start)


class Overload(object):

    def __init__(self, url, method, concurrency, numbers, duration, **options):
        self.url = url
        self.method = method
        self.concurrency = concurrency
        self.numbers = numbers
        self.duration = duration
        self.options = options

    @property
    def informations(self):
        res = requests.head(self.url)
        sys.stdout.write('\nServer: %(server)s\n' % res.headers)
        sys.stdout.write('URL: {}\n'.format(self.url))
        sys.stdout.write('Concurrency level: {}\n'.format(self.concurrency))
        sys.stdout.write('Options: {}\n\n'.format(self.options))

    @property
    def run(self):
        try:
            self.method = getattr(requests, self.method.lower())

            start = time.time()
            sys.stdout.flush()
            sys.stdout.write('[')

            pool = Pool(self.concurrency)

            if self.duration:
                with gevent.Timeout(self.duration, False):
                    while True:
                        pool.spawn(call, self.method, self.url, self.options)
                    pool.join()
            else:
                for number in xrange(self.numbers):
                    pool.spawn(call, self.method, self.url, self.options)
                pool.join()

            self.time_process = time.time() - start
            sys.stdout.flush()
            sys.stdout.write(']')
        except Exception as error:
            sys.stdout.write('error during run process ({})\n'.format(error))
            exit(1)

    @property
    def stats(self):
        try:
            self.total = sum([len(_stats[key]) for key in _stats.iterkeys()])
            self.total_success = len(_stats[200])
            if self.total_success:
                self.min = min(_stats[200])
                self.max = max(_stats[200])
                self.moy = sum(_stats[200]) / self.total_success
            else:
                self.min = 0
                self.max = 0
                self.moy = 0
            self.total_failed = self.total - self.total_success
            self.RPS = self.total_success / self.time_process
        except Exception as error:
            sys.stdout.write('error during stats process ({})\n'.format(error))
            pass
<commit_msg>improve the accuracy of stats<commit_after>#/usr/bin/env python
# -*- coding: utf-8 -*-

from gevent import monkey
monkey.patch_all()

import time
import requests
import argparse
import gevent
import os
import sys

from gevent.pool import Pool
from collections import defaultdict
from sys import exit


HTTP_VERBS = 'GET', 'POST', 'PUT', 'DELETE'
_stats = defaultdict(list)


def progress(func):
    """
    @decorator:
        display an tiret for each query treatment
    """
    def wrapper(*args):
        sys.stdout.write('-')
        sys.stdout.flush()
        return func(*args)
    return wrapper


def cookies_parse(cookies):
    """
    @args:
        <list_of_cookies>

    @return:
        <dict_of_cookies>
    """
    _cookies = {}
    try:
        for cookie in cookies:
            _cookie = cookie.split(':')
            if len(_cookie) != 2:
                raise Exception
            else:
                _cookie = iter(reversed(_cookie))
                _cookies[_cookie.next()] = _cookie.next()
    except Exception:
        sys.stdout.write('discarding invalid cookies: {}\n'.format(cookie))
        exit(1)
    else:
        return _cookies


def clear_stats():
    """
    keeps consistent stats when --repeat is used
    """
    global _stats
    del _stats
    _stats = defaultdict(list)


# TODO: fix bug AttributeError: 'Greenlet' object has no attribute '_run'
@progress
def call(method, url, options):
    """
    @args:
        <method_of_request>
        <str_url>
        <dict_requests_options>

    made a request and updates the stats
    """
    try:
        start = time.time()
        res = method(url, **options)
        code = res.status_code
    except requests.exceptions.Timeout:
        code = 408
    except Exception:
        code = 404
    finally:
        _stats[code].append(time.time()-start)


class Overload(object):

    def __init__(self, url, method, concurrency, numbers, duration, **options):
        self.url = url
        self.method = method
        self.concurrency = concurrency
        self.numbers = numbers
        self.duration = duration
        self.options = options

    @property
    def informations(self):
        res = requests.head(self.url)
        sys.stdout.write('\nServer: %(server)s\n' % res.headers)
        sys.stdout.write('URL: {}\n'.format(self.url))
        sys.stdout.write('Concurrency level: {}\n'.format(self.concurrency))
        sys.stdout.write('Options: {}\n\n'.format(self.options))

    @property
    def run(self):
        try:
            self.method = getattr(requests, self.method.lower())

            start = time.time()
            sys.stdout.flush()
            sys.stdout.write('[')

            pool = Pool(self.concurrency)

            if self.duration:
                with gevent.Timeout(self.duration, False):
                    while True:
                        pool.spawn(call, self.method, self.url, self.options)
                    pool.join()
            else:
                for number in xrange(self.numbers):
                    pool.spawn(call, self.method, self.url, self.options)
                pool.join()

            self.time_process = time.time() - start
            sys.stdout.flush()
            sys.stdout.write(']')
        except Exception as error:
            sys.stdout.write('error during run process ({})\n'.format(error))
            exit(1)

    @property
    def stats(self):
        try:
            self.total = sum([len(_stats[key]) for key in _stats.iterkeys()])
            self.total_success = len(_stats[200])
            if self.total_success:
                self.requests_process = sum(_stats[200])
                self.min = min(_stats[200])
                self.max = max(_stats[200])
                self.moy = self.requests_process / self.total_success
            else:
                self.requests_process = 0
                self.min = 0
                self.max = 0
                self.moy = 0
            self.total_failed = self.total - self.total_success
            self.RPS = self.total_success / self.requests_process
        except Exception as error:
            sys.stdout.write('error during stats process ({})\n'.format(error))
            pass

{'fraction_non_alphanumeric': 0.11467339097022095, 'fraction_numerical': 0.006123919308357349, 'mean_word_length': 1.8523972602739727, 'pattern_counts': {'":': 0, '<': 13, '<?xml version=': 0, '>': 13, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2089620', 'n_tokens_mistral': 2442, 'n_tokens_neox': 2334, 'n_words': 572}