<commit_before>#!/usr/bin/env python
__encoding__ = "utf-8"

import os
import pytoml as toml

class Trigger(object):
    """
    This class provides a filter to test a string against.
    """
    def __init__(self, config):
        self.config = config


        self.goodlistpath = config['trigger']['goodlist_path']
        self.goodlist = self.get_lists(self.goodlistpath)
        self.goodlist = self.strings_ok(self.goodlist)

        self.blacklistpath = config['trigger']['blacklist_path']
        self.blacklist = self.get_lists(self.blacklistpath)
        self.blacklist = self.strings_ok(self.blacklist)

    def get_lists(self, path):
        """
        pass a folder with text files in it. each line in the files becomes a filter word.

        :param path: path to folder whose files shall be added to the set
        :return: set of trigger words.
        """
        trigger_words = set()
        for filename in os.listdir(path):
            with open(path + filename, "r+") as f:
                [trigger_words.add(s.strip()) for s in f.readlines()]
        return trigger_words

    def strings_ok(self, filterlist):
        """
        Checks if an empty line is in a list and removes it.
        :param filterlist: a good- or blacklist.
        :return: filterlist: a corrected list.
        """
        for word in filterlist:
            if word == "\n":
                del word
        return filterlist

    def check_string(self, string):
        """
        checks if a string contains no bad words and at least 1 good word.

        :param string: A given string. Tweet or Toot, cleaned from html.
        :return: If the string passes the test
        """
        string = unicode.decode(string)
        for triggerword in self.goodlist:
            if string.lower().find(triggerword) != -1:
                for triggerword in self.blacklist:
                    if string.lower().find(triggerword) != -1:
                        return False
                return True
        return False

    def add_to_list(self, word, whichlist):
        """<commit_msg>list comprehension not needed anymore<commit_after>#!/usr/bin/env python
__encoding__ = "utf-8"

import os
import pytoml as toml

class Trigger(object):
    """
    This class provides a filter to test a string against.
    """
    def __init__(self, config):
        self.config = config


        self.goodlistpath = config['trigger']['goodlist_path']
        self.goodlist = self.get_lists(self.goodlistpath)
        self.goodlist = self.strings_ok(self.goodlist)

        self.blacklistpath = config['trigger']['blacklist_path']
        self.blacklist = self.get_lists(self.blacklistpath)
        self.blacklist = self.strings_ok(self.blacklist)

    def get_lists(self, path):
        """
        pass a folder with text files in it. each line in the files becomes a filter word.

        :param path: path to folder whose files shall be added to the set
        :return: set of trigger words.
        """
        trigger_words = set()
        for filename in os.listdir(path):
            with open(path + filename, "r+") as f:
                for s in f.readlines():
                    trigger_words.add(s.strip())
        return trigger_words

    def strings_ok(self, filterlist):
        """
        Checks if an empty line is in a list and removes it.
        :param filterlist: a good- or blacklist.
        :return: filterlist: a corrected list.
        """
        for word in filterlist:
            if word == "\n":
                del word
        return filterlist

    def check_string(self, string):
        """
        checks if a string contains no bad words and at least 1 good word.

        :param string: A given string. Tweet or Toot, cleaned from html.
        :return: If the string passes the test
        """
        string = unicode.decode(string)
        for triggerword in self.goodlist:
            if string.lower().find(triggerword) != -1:
                for triggerword in self.blacklist:
                    if string.lower().find(triggerword) != -1:
                        return False
                return True
        return False

    def add_to_list(self, word, whichlist):
        """
{'fraction_non_alphanumeric': 0.09207100591715976, 'fraction_numerical': 0.001893491124260355, 'mean_word_length': 1.9865724381625443, 'pattern_counts': {'":': 2, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6454161', 'n_tokens_mistral': 1174, 'n_tokens_neox': 1134, 'n_words': 392}