<commit_before>import json
import sys
import traceback
import time

from tokenserver import logger
from tokenserver.crypto.master import Response, PROTOBUF_CLASSES

from browserid._m2_monkeypatch import DSA as _DSA
from browserid._m2_monkeypatch import RSA as _RSA
from browserid import jwt
from browserid.certificates import CertificatesManager
from browserid.tests.support import fetch_public_key

from M2Crypto import BIO


class ExpiredValue(KeyError):
    pass


class TTLedDict(object):
    """A simple TTLed in memory cache.

    :param ttl: the time-to-leave for records, in seconds.
                The cache will return an ExpiredValue once the TTL is over for
                its records, and remove the items from its cache.
    """

    def __init__(self, ttl, storage=None):
        if storage is None:
            storage = {}

        self.ttl = ttl
        self._storage = storage

    def __setitem__(self, key, value):
        self._storage[key] = time.time(), value

    def __getitem__(self, key):
        insert_date, value = self._storage[key]
        if insert_date != 0 and insert_date + self.ttl < time.time():
            # if the ttl is expired, remove the key from the cache and return a
            # key error
            del self._storage[key]
            raise ExpiredValue(key)
        return value

    def set_ttl(self, key, ttl):
        _, value = self._storage[key]
        self._storage[key] = ttl, value


class CertificatesManagerWithCache(CertificatesManager):

    def __init__(self, memory=None, memcache=None, loadtest_mode=False):
        """If the loadtest mode is set, when looking for loadtest.local, the
        certificate bundled in browserid.tests.support will be returned instead
        of failing."""
        if memory is None:
            memory = TTLedDict(60 * 10)  # TTL of 10 minutes for the certs

        self.memory = memory
        self.memcache = memcache

        if loadtest_mode is True:
            self.memory['loadtest.local'] = fetch_public_key('loadtest.local')

    def __getitem__(self, hostname):
        """Get the certificate for the given hostname.

        If the certificate is not already in memory, try to get it in the
        shared memcache. If it's not in the the memcache, download it and store
        it both in memory and in the memcache.
        """
        try:
            # Use a cached key if available.
            return self.memory[hostname]
        except KeyError:
            # try to get the key from memcache if it doesn't exist in memory
                try:
                    # supposely the memcache lookup failed
                    raise KeyError()
                    key = self.memcache.get(hostname)
                    self.memory[hostname] = key
                    return key
                except KeyError:
                    # it doesn't exist in memcache either, so let's get it from<commit_msg>we don't want the loadtest mode cert to expire<commit_after>import json
import sys
import traceback
import time

from tokenserver import logger
from tokenserver.crypto.master import Response, PROTOBUF_CLASSES

from browserid._m2_monkeypatch import DSA as _DSA
from browserid._m2_monkeypatch import RSA as _RSA
from browserid import jwt
from browserid.certificates import CertificatesManager
from browserid.tests.support import fetch_public_key

from M2Crypto import BIO


class ExpiredValue(KeyError):
    pass


class TTLedDict(object):
    """A simple TTLed in memory cache.

    :param ttl: the time-to-leave for records, in seconds.
                The cache will return an ExpiredValue once the TTL is over for
                its records, and remove the items from its cache.

                A ttl of 0 means the record never expires.
    """

    def __init__(self, ttl, storage=None):
        if storage is None:
            storage = {}

        self.ttl = ttl
        self._storage = storage

    def __setitem__(self, key, value):
        self._storage[key] = time.time(), value

    def __getitem__(self, key):
        insert_date, value = self._storage[key]
        if insert_date != 0 and insert_date + self.ttl < time.time():
            # if the ttl is expired, remove the key from the cache and return a
            # key error
            del self._storage[key]
            raise ExpiredValue(key)
        return value

    def set_ttl(self, key, ttl):
        _, value = self._storage[key]
        self._storage[key] = ttl, value


class CertificatesManagerWithCache(CertificatesManager):

    def __init__(self, memory=None, memcache=None, loadtest_mode=False):
        """If the loadtest mode is set, when looking for loadtest.local, the
        certificate bundled in browserid.tests.support will be returned instead
        of failing."""
        if memory is None:
            memory = TTLedDict(60 * 10)  # TTL of 10 minutes for the certs

        self.memory = memory
        self.memcache = memcache

        if loadtest_mode is True:
            self.memory['loadtest.local'] = fetch_public_key('loadtest.local')
            self.memory.set_ttl('loadtest.local', 0)  # never expire

    def __getitem__(self, hostname):
        """Get the certificate for the given hostname.

        If the certificate is not already in memory, try to get it in the
        shared memcache. If it's not in the the memcache, download it and store
        it both in memory and in the memcache.
        """
        try:
            # Use a cached key if available.
            return self.memory[hostname]
        except KeyError:
            # try to get the key from memcache if it doesn't exist in memory
                try:
                    # supposely the memcache lookup failed
                    raise KeyError()
                    key = self.memcache.get(hostname)
                    self.memory[hostname] = key
                    return key
                except KeyError:
                    # it doesn't exist in memcache either, so let's get it from
{'fraction_non_alphanumeric': 0.06516666666666666, 'fraction_numerical': 0.0036666666666666666, 'mean_word_length': 2.1903242955874536, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3604089', 'n_tokens_mistral': 1662, 'n_tokens_neox': 1577, 'n_words': 614}