<commit_before>import tempfile
import shutil
import atexit

from zipfile import ZipFile

from zope.interface import implements
from scrapy.interfaces import ISpiderManager
from scrapy.utils.misc import load_object

from slybot.spider import IblSpider
from slybot.utils import open_project_from_dir, load_plugins


class SlybotSpiderManager(object):

    implements(ISpiderManager)

    def __init__(self, datadir, spider_cls=None, settings=None, **kwargs):
        self.spider_cls = load_object(spider_cls) if spider_cls else IblSpider
        self._specs = open_project_from_dir(datadir)
        settings = settings.copy()
        settings.set('PLUGINS', load_plugins(settings))
        self.settings = settings

    @classmethod
    def from_crawler(cls, crawler):
        # backwards compatibility with Scrapy < 0.25
        return cls.from_settings(crawler.settings)

    @classmethod
    def from_settings(cls, settings):
        datadir = settings['PROJECT_DIR']
        spider_cls = settings['SLYBOT_SPIDER_CLASS']
        return cls(datadir, spider_cls, settings=settings)

    def load(self, spider_name):
        spec = self._specs["spiders"][spider_name]
        items = self._specs["items"]
        extractors = self._specs["extractors"]

        class SlybotSpider(self.spider_cls):
            def __init__(self, **kwargs):
                super(SlybotSpider, self).__init__(spider_name, spec, items,
                                                   extractors, self.settings,
                                                   **kwargs)

        return SlybotSpider
<commit_msg>fix slybot under scrapy 0.25<commit_after>import tempfile
import shutil
import atexit

from zipfile import ZipFile

from zope.interface import implements
from scrapy.interfaces import ISpiderManager
from scrapy.utils.misc import load_object

from slybot.spider import IblSpider
from slybot.utils import open_project_from_dir, load_plugins


class SlybotSpiderManager(object):

    implements(ISpiderManager)

    def __init__(self, datadir, spider_cls=None, settings=None, **kwargs):
        self.spider_cls = load_object(spider_cls) if spider_cls else IblSpider
        self._specs = open_project_from_dir(datadir)
        settings = settings.copy()
        settings.frozen = False
        settings.set('PLUGINS', load_plugins(settings))
        self.settings = settings

    @classmethod
    def from_crawler(cls, crawler):
        # backwards compatibility with Scrapy < 0.25
        return cls.from_settings(crawler.settings)

    @classmethod
    def from_settings(cls, settings):
        datadir = settings['PROJECT_DIR']
        spider_cls = settings['SLYBOT_SPIDER_CLASS']
        return cls(datadir, spider_cls, settings=settings)

    def load(self, spider_name):
        spec = self._specs["spiders"][spider_name]
        items = self._specs["items"]
        extractors = self._specs["extractors"]

        class SlybotSpider(self.spider_cls):
            def __init__(self_, **kwargs):
                super(SlybotSpider, self_).__init__(spider_name, spec, items,
                                                   extractors, self.settings,
                                                   **kwargs)

        return SlybotSpider

{'fraction_non_alphanumeric': 0.07996295152824946, 'fraction_numerical': 0.002778635381290522, 'mean_word_length': 2.7674418604651163, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 8, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3926971', 'n_tokens_mistral': 966, 'n_tokens_neox': 930, 'n_words': 206}