<commit_before>import operator
from itertools import imap, repeat

from spicedham.plugin import BasePlugin

class NonsenseFilter(BasePlugin):
    """
    Filter messages with no words in the database.
    """

    def __init__(self, config, backend):
        """
        Get values from the config.
        """
        self.backend = backend
        nonsensefilter_config = config.get('nonsensefilter', {})
        self.filter_match = nonsensefilter_config.get('filter_match', 1)
        self.filter_miss = nonsensefilter_config.get('filter_miss', None)
    
    def train(self, response, value):
        """
        Set each word to True.
        """
        self.backend.set_key_list(self.__class__.__name__, {(word, True) for word in response})

    # TODO: Will match responses consisting of only ''
    def classify(self, response):
        """
        If the message contains only words not found in the database return
        filter_match. Else return filter_miss.
        """
        classifier = self.__class__.__name__
        list_in_dict = lambda x, y: not self.backend.get_key(x, y, False)
        if all(imap(list_in_dict, repeat(classifier), response)):
            return self.filter_match
        else:
            return self.filter_miss<commit_msg>No set comprehensions in python 2.6 :(<commit_after>import operator
from itertools import imap, repeat

from spicedham.plugin import BasePlugin

class NonsenseFilter(BasePlugin):
    """
    Filter messages with no words in the database.
    """

    def __init__(self, config, backend):
        """
        Get values from the config.
        """
        self.backend = backend
        nonsensefilter_config = config.get('nonsensefilter', {})
        self.filter_match = nonsensefilter_config.get('filter_match', 1)
        self.filter_miss = nonsensefilter_config.get('filter_miss', None)
    
    def train(self, response, value):
        """
        Set each word to True.
        """
        self.backend.set_key_list(self.__class__.__name__, set([(word, True) for word in response]))

    # TODO: Will match responses consisting of only ''
    def classify(self, response):
        """
        If the message contains only words not found in the database return
        filter_match. Else return filter_miss.
        """
        classifier = self.__class__.__name__
        list_in_dict = lambda x, y: not self.backend.get_key(x, y, False)
        if all(imap(list_in_dict, repeat(classifier), response)):
            return self.filter_match
        else:
            return self.filter_miss
{'fraction_non_alphanumeric': 0.09392049883086516, 'fraction_numerical': 0.001558846453624318, 'mean_word_length': 2.605337078651685, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3367602', 'n_tokens_mistral': 705, 'n_tokens_neox': 693, 'n_words': 207}