<commit_before># -*- coding: utf-8 -*-
import os
import re
from time import sleep
import datetime

import pytest
from mock import patch, MagicMock
from scrapy import Item
from scrapy.exceptions import DontCloseSpider
from scrapy.http import Response
from scrapy.settings import Settings
from scrapy.utils.test import get_crawler
from twisted.internet.defer import Deferred
from twisted.python.failure import Failure
from twisted.trial import unittest
from twisted.web.error import Error

from scrapyrt.core import CrawlManager
from scrapyrt.conf import settings

from .spiders import MetaSpider


class TestCrawlManager(unittest.TestCase):

    def setUp(self):
        self.url = 'http://localhost'
        self.kwargs = {'url': self.url, 'dont_filter': True}
        self.crawler = MagicMock()
        self.spider = MetaSpider.from_crawler(self.crawler)
        self.crawler.spider = self.spider
        self.crawl_manager = self.create_crawl_manager()
        self.crawl_manager.crawler = self.crawler
        self.item = Item()
        self.response = Response('http://localhost')
        self.another_spider = MetaSpider.from_crawler(self.crawler)

    def create_crawl_manager(self, kwargs=None):
        kwargs = kwargs if kwargs else self.kwargs.copy()
        crawl_manager = CrawlManager(self.spider.name, kwargs)
        crawl_manager.crawler = self.crawler
        return crawl_manager


@patch('scrapyrt.core.ScrapyrtCrawlerProcess.crawl', return_value=Deferred())
class TestCrawl(TestCrawlManager):

    def test_crawl(self, crawler_process_mock):
        result = self.crawl_manager.crawl()
        self.assertIsInstance(result, Deferred)
        self.assertGreater(len(result.callbacks), 0)
        self.assertEqual(
            result.callbacks[0][0][0], self.crawl_manager.return_items)

    def test_no_spider(self, crawler_process_mock):
        # spider wasn't found
        crawler_process_mock.side_effect = KeyError
        exception = self.assertRaises(
            Error, self.crawl_manager.crawl)
        self.assertTrue(crawler_process_mock.called)
        self.assertEqual(exception.status, '404')

    def test_spider_exists(self, crawler_process_mock):
        result = self.crawl_manager.crawl()
        self.assertTrue(crawler_process_mock.called)
        self.assertIs(result, crawler_process_mock.return_value)

    def test_spider_arguments_are_passed(self, crawler_process_mock):
        spider_args = ['a', 'b']
        spider_kwargs = {'a': 1, 'b': 2}
        self.crawl_manager.crawl(*spider_args, **spider_kwargs)
        self.assertTrue(crawler_process_mock.called)
        call_args, call_kwargs = crawler_process_mock.call_args
        for arg in spider_args:
            self.assertIn(arg, call_args)
        self.assertDictContainsSubset(spider_kwargs, call_kwargs)


class TestGetProjectSettings(TestCrawlManager):

    def test_get_project_settings(self):
        result = self.crawl_manager.get_project_settings()
        self.assertIsInstance(result, Settings)<commit_msg>Fix deprecated feature<commit_after># -*- coding: utf-8 -*-
import os
import re
from time import sleep
import datetime

import pytest
from mock import patch, MagicMock
from scrapy import Item
from scrapy.exceptions import DontCloseSpider
from scrapy.http import Response
from scrapy.settings import Settings
from scrapy.utils.test import get_crawler
from twisted.internet.defer import Deferred
from twisted.python.failure import Failure
from twisted.trial import unittest
from twisted.web.error import Error

from scrapyrt.core import CrawlManager
from scrapyrt.conf import settings

from .spiders import MetaSpider


class TestCrawlManager(unittest.TestCase):

    def setUp(self):
        self.url = 'http://localhost'
        self.kwargs = {'url': self.url, 'dont_filter': True}
        self.crawler = MagicMock()
        self.spider = MetaSpider.from_crawler(self.crawler)
        self.crawler.spider = self.spider
        self.crawl_manager = self.create_crawl_manager()
        self.crawl_manager.crawler = self.crawler
        self.item = Item()
        self.response = Response('http://localhost')
        self.another_spider = MetaSpider.from_crawler(self.crawler)

    def create_crawl_manager(self, kwargs=None):
        kwargs = kwargs if kwargs else self.kwargs.copy()
        crawl_manager = CrawlManager(self.spider.name, kwargs)
        crawl_manager.crawler = self.crawler
        return crawl_manager


@patch('scrapyrt.core.ScrapyrtCrawlerProcess.crawl', return_value=Deferred())
class TestCrawl(TestCrawlManager):

    def test_crawl(self, crawler_process_mock):
        result = self.crawl_manager.crawl()
        self.assertIsInstance(result, Deferred)
        self.assertGreater(len(result.callbacks), 0)
        self.assertEqual(
            result.callbacks[0][0][0], self.crawl_manager.return_items)

    def test_no_spider(self, crawler_process_mock):
        # spider wasn't found
        crawler_process_mock.side_effect = KeyError
        exception = self.assertRaises(
            Error, self.crawl_manager.crawl)
        self.assertTrue(crawler_process_mock.called)
        self.assertEqual(exception.status, '404')

    def test_spider_exists(self, crawler_process_mock):
        result = self.crawl_manager.crawl()
        self.assertTrue(crawler_process_mock.called)
        self.assertIs(result, crawler_process_mock.return_value)

    def test_spider_arguments_are_passed(self, crawler_process_mock):
        spider_args = ['a', 'b']
        spider_kwargs = {'a': 1, 'b': 2}
        self.crawl_manager.crawl(*spider_args, **spider_kwargs)
        self.assertTrue(crawler_process_mock.called)
        call_args, call_kwargs = crawler_process_mock.call_args
        for arg in spider_args:
            self.assertIn(arg, call_args)
        self.assertEqual(spider_kwargs, call_kwargs)


class TestGetProjectSettings(TestCrawlManager):

    def test_get_project_settings(self):
        result = self.crawl_manager.get_project_settings()
        self.assertIsInstance(result, Settings)
{'fraction_non_alphanumeric': 0.08874172185430464, 'fraction_numerical': 0.0033112582781456954, 'mean_word_length': 3.6149732620320854, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5188411', 'n_tokens_mistral': 1924, 'n_tokens_neox': 1849, 'n_words': 367}