<commit_before># -*- coding: utf-8 -*-

from collections import OrderedDict

from twisted.internet import reactor


# needed in order to allow UT override
test_reactor = None


class TempDict(OrderedDict):
    reactor = None
    expireCallback = None

    def __init__(self, timeout=None, size_limit=None):
        self.timeout = timeout
        self.size_limit = size_limit
        OrderedDict.__init__(self)

        self.reactor = reactor if test_reactor is None else test_reactor

        self._check_size_limit()

    def get_timeout(self):
        """The override of this method allows dynamic limits imlementations"""
        return self.timeout

    def get_size_limit(self):
        """The override of this method allows dynamic limits imlementations"""
        return self.size_limit

    def set(self, key, value):
        timeout = self.get_timeout()
        if timeout is not None:
            value._expireCall = self.reactor.callLater(timeout, self._expire, key)
        else:
            value._expireCall = None

        self[key] = value

        self._check_size_limit()

    def get(self, key):
        item = self[key] if key in self else None

        if item is not None and item._expireCall is not None:
            item._expireCall.reset(self.get_timeout())

        return item

    def _check_size_limit(self):
        size_limit = self.get_size_limit()
        if size_limit is not None:
            while len(self) > size_limit:
                self.popitem(last=False)

    def _expire(self, key):
        try:
            if self.expireCallback is not None:
                self.expireCallback(self[key])
        except Exception as e:
          print e
          pass

        try:
            del self[key]
        except:
            pass
<commit_msg>Avoid shadowing exceptions in utils/tempdict.py<commit_after># -*- coding: utf-8 -*-

from collections import OrderedDict

from twisted.internet import reactor


# needed in order to allow UT override
test_reactor = None


class TempDict(OrderedDict):
    reactor = None
    expireCallback = None

    def __init__(self, timeout=None, size_limit=None):
        self.timeout = timeout
        self.size_limit = size_limit
        OrderedDict.__init__(self)

        self.reactor = reactor if test_reactor is None else test_reactor

        self._check_size_limit()

    def get_timeout(self):
        """The override of this method allows dynamic limits imlementations"""
        return self.timeout

    def get_size_limit(self):
        """The override of this method allows dynamic limits imlementations"""
        return self.size_limit

    def set(self, key, value):
        timeout = self.get_timeout()
        if timeout is not None:
            value._expireCall = self.reactor.callLater(timeout, self._expire, key)
        else:
            value._expireCall = None

        self[key] = value

        self._check_size_limit()

    def get(self, key):
        item = self[key] if key in self else None

        if item is not None and item._expireCall is not None:
            item._expireCall.reset(self.get_timeout())

        return item

    def _check_size_limit(self):
        size_limit = self.get_size_limit()
        if size_limit is not None:
            while len(self) > size_limit:
                self.popitem(last=False)

    def _expire(self, key):
        if self.expireCallback is not None:
            self.expireCallback(self[key])

        del self[key]

{'fraction_non_alphanumeric': 0.0778322283078697, 'fraction_numerical': 0.0005765350245027386, 'mean_word_length': 2.2735849056603774, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3381427', 'n_tokens_mistral': 1004, 'n_tokens_neox': 961, 'n_words': 293}