<commit_before>from time import clock
import logging

from lxml import html
import requests
import simplejson

from . import helpers
from .settings import DELAY, USERNAME, PASSWORD


log = logging.getLogger(__name__)


class Session(requests.Session):

    @classmethod
    def login(cls, username=USERNAME, password=PASSWORD, headers=None):
        session = cls()
        helpers.login(session, {'username': username, 'password': password}, headers)
        return session

    def __init__(self, *args, **kwargs):
        super().__init__(*args, **kwargs)
        self.timestamp = -DELAY

    def post(self, *args, **kwargs):
        while clock() - self.timestamp < DELAY:
            pass
        self.timestamp = clock()
        response = super().post(*args, **kwargs)
        response.raise_for_status()
        return response

    def get(self, *args, **kwargs):
        while clock() - self.timestamp < DELAY:
            pass
        self.timestamp = clock()
        response = super().get(*args, **kwargs)
        response.raise_for_status()
        # log.debug(simplejson.dumps({'method': 'GET', 'args': args, 'kwargs': kwargs, 'content': response.content.decode('utf8')}))
        return response


class Question(object):

    def __init__(self, text, user_answer, explanation):
        self.text = text<commit_msg>Make throttling mechanism better, remove commented loggin.<commit_after>import time
import logging

from lxml import html
import requests

from . import helpers
from .settings import DELAY, USERNAME, PASSWORD


log = logging.getLogger(__name__)


class Session(requests.Session):

    @classmethod
    def login(cls, username=USERNAME, password=PASSWORD, headers=None):
        session = cls()
        helpers.login(session, {'username': username, 'password': password}, headers)
        return session

    def __init__(self, *args, **kwargs):
        super().__init__(*args, **kwargs)
        self.timestamp = -DELAY

    def _throttle(self):
        while time.clock() - self.timestamp < DELAY:
            time.sleep(.5)
        self.timestamp = time.clock()

    def post(self, *args, **kwargs):
        self._throttle()
        response = super().post(*args, **kwargs)
        response.raise_for_status()
        return response

    def get(self, *args, **kwargs):
        self._throttle()
        response = super().get(*args, **kwargs)
        response.raise_for_status()
        return response


class Question(object):

    def __init__(self, text, user_answer, explanation):
        self.text = text
{'fraction_non_alphanumeric': 0.12151002752654345, 'fraction_numerical': 0.0007864726700747149, 'mean_word_length': 2.9564541213063764, 'pattern_counts': {'":': 0, '<': 6, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 2}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2080164', 'n_tokens_mistral': 725, 'n_tokens_neox': 704, 'n_words': 194}