<commit_before>import inspect
import os
import re
from FourmiCrawler.parsers.parser import Parser


class SourceLoader:
    sources = []

    def __init__(self, rel_dir="FourmiCrawler/parsers"):
        path = os.path.dirname(os.path.abspath(__file__))
        path += "/" + rel_dir
        known_parser = set()

        for py in [f[:-3] for f in os.listdir(path) if f.endswith('.py') and f != '__init__.py']:
            mod = __import__('.'.join([rel_dir.replace("/", "."), py]), fromlist=[py])
            classes = [getattr(mod, x) for x in dir(mod) if inspect.isclass(getattr(mod, x))]
            for cls in classes:
                if issubclass(cls, Parser) and cls not in known_parser:
                    self.sources.append(cls())  # [review] - Would we ever need arguments for the parsers?
                    known_parser.add(cls)

    def include(self, source_names):
        new = []
        for name in source_names:
            new.extend([src for src in self.sources if re.match(name, src.__class__.__name__)])
        self.sources = new

    def exclude(self, source_names):
        exclude = []
        for name in source_names:
           exclude.extend([src for src in self.sources if re.match(source_names,src.__class__.__name__)])
        self.sources = [src for src in self.sources if src not in exclude]

    def __str__(self):
        string = ""
        for src in self.sources:
            string += "Source: " + src.__class__.__name__
            string += " - "
            string +=  "URI: " + src.website + "\n"
        return string<commit_msg>Include and Exclude fix after testing<commit_after>import inspect
import os
import re
from FourmiCrawler.parsers.parser import Parser


class SourceLoader:
    sources = []

    def __init__(self, rel_dir="FourmiCrawler/parsers"):
        path = os.path.dirname(os.path.abspath(__file__))
        path += "/" + rel_dir
        known_parser = set()

        for py in [f[:-3] for f in os.listdir(path) if f.endswith('.py') and f != '__init__.py']:
            mod = __import__('.'.join([rel_dir.replace("/", "."), py]), fromlist=[py])
            classes = [getattr(mod, x) for x in dir(mod) if inspect.isclass(getattr(mod, x))]
            for cls in classes:
                if issubclass(cls, Parser) and cls not in known_parser:
                    self.sources.append(cls())  # [review] - Would we ever need arguments for the parsers?
                    known_parser.add(cls)

    def include(self, source_names):
        print source_names
        new = set()
        for name in source_names:
            new.update([src for src in self.sources if re.match(name, src.__class__.__name__)])
        self.sources = list(new)

    def exclude(self, source_names):
        exclude = []
        for name in source_names:
            exclude.extend([src for src in self.sources if re.match(name, src.__class__.__name__)])
        self.sources = [src for src in self.sources if src not in exclude]

    def __str__(self):
        string = ""
        for src in self.sources:
            string += "Source: " + src.__class__.__name__
            string += " - "
            string += "URI: " + src.website + "\n"
        return string
{'fraction_non_alphanumeric': 0.1214574898785425, 'fraction_numerical': 0.0006228589224540642, 'mean_word_length': 2.221664994984955, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5582759', 'n_tokens_mistral': 974, 'n_tokens_neox': 951, 'n_words': 274}