<commit_before>import os
import glob


__all__ = [os.path.basename(f)[:-3]
           for f in glob.glob(os.path.dirname(__file__) + "/*.py")]


def _import_all_extractor_files():
    __import__(__name__, globals(), locals(), __all__, 0)


def get_extractor_instances():
    """ Get all classes with names ending with 'Extractor' from
    the module extractors directory
    instantiate it and return a list with all the instances
    """
    _import_all_extractor_files()

    extractor_instance_list = []
    for mod in [v for k, v in globals().items() if k in __all__]:
        extractor_instance_list += [getattr(mod, klass)()
                                    for klass in dir(mod)
                                    if klass.endswith("Extractor")
                                    and klass != "BaseExtractor"]
    return extractor_instance_list


def get_extractor(name):
    for extractor in get_extractor_instances():
        if extractor.name == name:
            return extractor


def get_from_url(url):
    for extractor in get_extractor_instances():
        if extractor.is_valid_url(url):
            return extractor<commit_msg>Added method to get extractor from a host name to extractors module<commit_after>import os
import glob


__all__ = [os.path.basename(f)[:-3]
           for f in glob.glob(os.path.dirname(__file__) + "/*.py")]


def _import_all_extractor_files():
    __import__(__name__, globals(), locals(), __all__, 0)


def get_extractor_instances():
    """ Get all classes with names ending with 'Extractor' from
    the module extractors directory
    instantiate it and return a list with all the instances
    """
    _import_all_extractor_files()

    extractor_instance_list = []
    for mod in [v for k, v in globals().items() if k in __all__]:
        extractor_instance_list += [getattr(mod, klass)()
                                    for klass in dir(mod)
                                    if klass.endswith("Extractor")
                                    and klass != "BaseExtractor"]
    return extractor_instance_list


def get_extractor(name):
    for extractor in get_extractor_instances():
        if extractor.name == name:
            return extractor


def get_from_host(host):
    for extractor in get_extractor_instances():
        if host in extractor.host_list:
            return extractor


def get_from_url(url):
    for extractor in get_extractor_instances():
        if extractor.is_valid_url(url):
            return extractor
{'fraction_non_alphanumeric': 0.08733974358974358, 'fraction_numerical': 0.0016025641025641025, 'mean_word_length': 2.4632454923717058, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 6, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3369157', 'n_tokens_mistral': 731, 'n_tokens_neox': 711, 'n_words': 208}