<commit_before># -*- coding: utf-8 -*-


from mydm.spider import register_meta_classes
from mydm.exceptions import SpiderFactoryException


__all__ = ['SpiderFactory']


class SpiderFactory:

    meta_classes = {}

    def __init_subclass__(cls, spider_type, **kwargs):
        super().__init_subclass__(**kwargs)
        cls.meta_classes[spider_type] = cls

    @classmethod
    def create_spider(cls, setting):
        if 'name' not in setting or 'type' not in setting:
            raise SpiderFactoryException(
                    'miss attribute[name|type]'
            )
        try:
            metacls = cls.meta_classes[setting['type']]
        except KeyError:
            raise SpiderFactoryException(
                    f'unknown spider type[{setting["type"]}]'
            )
        try:
            return metacls(
                    f'{setting["name"].capitalize()}Spider',
                    (),
                    setting
            )
        except AttributeError as e:
            raise SpiderFactoryException(f'build spider failed[{e}]')


register_meta_classes()<commit_msg>format<commit_after># -*- coding: utf-8 -*-


from mydm.spider import register_meta_classes
from mydm.exceptions import SpiderFactoryException


__all__ = ['SpiderFactory']


class SpiderFactory:

    META_CLASSES = {}

    def __init_subclass__(cls, spider_type, **kwargs):
        super().__init_subclass__(**kwargs)
        cls.META_CLASSES[spider_type] = cls

    @classmethod
    def create_spider(cls, setting):
        if 'name' not in setting or 'type' not in setting:
            raise SpiderFactoryException(
                    'miss attribute[name|type]'
            )
        try:
            metacls = cls.META_CLASSES[setting['type']]
        except KeyError:
            raise SpiderFactoryException(
                    f'unknown spider type[{setting["type"]}]'
            )
        try:
            return metacls(
                    f'{setting["name"].capitalize()}Spider',
                    (),
                    setting
            )
        except AttributeError as e:
            raise SpiderFactoryException(f'build spider failed[{e}]')


register_meta_classes()
{'fraction_non_alphanumeric': 0.09949794614331356, 'fraction_numerical': 0.0009128251939753537, 'mean_word_length': 1.9986320109439124, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7535895', 'n_tokens_mistral': 630, 'n_tokens_neox': 578, 'n_words': 129}