<commit_before>import six
from grab import Grab
from collections import defaultdict

import grab.spider.base
from grab import Grab
from grab.spider import Spider, Task, SpiderMisuseError, NoTaskHandler
from test.util import (BaseGrabTestCase, build_grab, build_spider,
                       multiprocess_mode)
from grab.spider.error import SpiderError
from weblib.error import ResponseNotValid


class SimpleSpider(Spider):
    def task_baz(self, grab, task):
        self.SAVED_ITEM = grab.response.body


class TestSpider(BaseGrabTestCase):
    def setUp(self):
        self.server.reset()

    def test_task_priority(self):
        # Automatic random priority
        grab.spider.base.RANDOM_TASK_PRIORITY_RANGE = (10, 20)
        bot = build_spider(SimpleSpider, priority_mode='random')
        bot.setup_queue()
        task = Task('baz', url='http://xxx.com')
        self.assertEqual(task.priority, None)
        bot.add_task(task)
        self.assertTrue(10 <= task.priority <= 20)

        # Automatic constant priority
        grab.spider.base.DEFAULT_TASK_PRIORITY = 33
        bot = build_spider(SimpleSpider, priority_mode='const')
        bot.setup_queue()
        task = Task('baz', url='http://xxx.com')
        self.assertEqual(task.priority, None)
        bot.add_task(task)
        self.assertEqual(33, task.priority)

        # Automatic priority does not override explictily setted priority
        grab.spider.base.DEFAULT_TASK_PRIORITY = 33
        bot = build_spider(SimpleSpider, priority_mode='const')
        bot.setup_queue()
        task = Task('baz', url='http://xxx.com', priority=1)
        self.assertEqual(1, task.priority)
        bot.add_task(task)
        self.assertEqual(1, task.priority)

        self.assertRaises(SpiderMisuseError,
                          lambda: SimpleSpider(priority_mode='foo'))

    def test_task_url(self):
        bot = build_spider(SimpleSpider, )
        bot.setup_queue()
        task = Task('baz', url='http://xxx.com')
        self.assertEqual('http://xxx.com', task.url)
        bot.add_task(task)
        self.assertEqual('http://xxx.com', task.url)
        self.assertEqual(None, task.grab_config)

        g = Grab(url='http://yyy.com')
        task = Task('baz', grab=g)
        bot.add_task(task)
        self.assertEqual('http://yyy.com', task.url)
        self.assertEqual('http://yyy.com', task.grab_config['url'])

    def test_task_clone(self):
        bot = build_spider(SimpleSpider, )
        bot.setup_queue()

        task = Task('baz', url='http://xxx.com')
        bot.add_task(task.clone())

        # Pass grab to clone
        task = Task('baz', url='http://xxx.com')
        g = Grab()
        g.setup(url='zzz')
        bot.add_task(task.clone(grab=g))

        # Pass grab_config to clone
        task = Task('baz', url='http://xxx.com')
        g = Grab()
        g.setup(url='zzz')
        bot.add_task(task.clone(grab_config=g.config))

    def test_task_clone_with_url_param(self):
        task = Task('baz', url='http://xxx.com')
        task.clone(url='http://yandex.ru/')

    def test_task_useragent(self):
        bot = build_spider(SimpleSpider, )
        bot.setup_queue()

        g = Grab()
        g.setup(url=self.server.get_url())
        g.setup(user_agent='Foo')

        task = Task('baz', grab=g)
        bot.add_task(task.clone())<commit_msg>Small fix in spider tests<commit_after>import six
from grab import Grab
from collections import defaultdict

import grab.spider.base
from grab import Grab
from grab.spider import Spider, Task, SpiderMisuseError, NoTaskHandler
from test.util import (BaseGrabTestCase, build_grab, build_spider,
                       multiprocess_mode)
from grab.spider.error import SpiderError
from weblib.error import ResponseNotValid


class SimpleSpider(Spider):
    def task_baz(self, grab, task):
        self.SAVED_ITEM = grab.response.body


class TestSpider(BaseGrabTestCase):
    def setUp(self):
        self.server.reset()

    def test_task_priority(self):
        # Automatic random priority
        grab.spider.base.RANDOM_TASK_PRIORITY_RANGE = (10, 20)
        bot = build_spider(SimpleSpider, priority_mode='random')
        bot.setup_queue()
        task = Task('baz', url='http://xxx.com')
        self.assertEqual(task.priority, None)
        bot.add_task(task)
        self.assertTrue(10 <= task.priority <= 20)

        # Automatic constant priority
        grab.spider.base.DEFAULT_TASK_PRIORITY = 33
        bot = build_spider(SimpleSpider, priority_mode='const')
        bot.setup_queue()
        task = Task('baz', url='http://xxx.com')
        self.assertEqual(task.priority, None)
        bot.add_task(task)
        self.assertEqual(33, task.priority)

        # Automatic priority does not override explictily setted priority
        grab.spider.base.DEFAULT_TASK_PRIORITY = 33
        bot = build_spider(SimpleSpider, priority_mode='const')
        bot.setup_queue()
        task = Task('baz', url='http://xxx.com', priority=1)
        self.assertEqual(1, task.priority)
        bot.add_task(task)
        self.assertEqual(1, task.priority)

        self.assertRaises(SpiderMisuseError,
                          lambda: SimpleSpider(priority_mode='foo'))

    def test_task_url(self):
        bot = build_spider(SimpleSpider, )
        bot.setup_queue()
        task = Task('baz', url='http://xxx.com')
        self.assertEqual('http://xxx.com', task.url)
        bot.add_task(task)
        self.assertEqual('http://xxx.com', task.url)
        self.assertEqual(None, task.grab_config)

        g = Grab(url='http://yyy.com')
        task = Task('baz', grab=g)
        bot.add_task(task)
        self.assertEqual('http://yyy.com', task.url)
        self.assertEqual('http://yyy.com', task.grab_config['url'])

    def test_task_clone(self):
        bot = build_spider(SimpleSpider, )
        bot.setup_queue()

        task = Task('baz', url='http://xxx.com')
        bot.add_task(task.clone())

        # Pass grab to clone
        task = Task('baz', url='http://xxx.com')
        g = Grab()
        g.setup(url='zzz')
        bot.add_task(task.clone(grab=g))

        # Pass grab_config to clone
        task = Task('baz', url='http://xxx.com')
        g = Grab()
        g.setup(url='zzz')
        bot.add_task(task.clone(grab_config=g.config))

    def test_task_clone_with_url_param(self):
        task = Task('baz', url='http://example.com/path')
        task2 = task.clone(url='http://example.com/new')
        self.assertEqual(task2.name, 'baz')
        self.assertEqual(task2.url, 'http://example.com/new')

    def test_task_useragent(self):
        bot = build_spider(SimpleSpider, )
        bot.setup_queue()

        g = Grab()
        g.setup(url=self.server.get_url())
        g.setup(user_agent='Foo')

        task = Task('baz', grab=g)
        bot.add_task(task.clone())
{'fraction_non_alphanumeric': 0.13134110787172013, 'fraction_numerical': 0.005393586005830904, 'mean_word_length': 2.6224920802534317, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6232179', 'n_tokens_mistral': 2227, 'n_tokens_neox': 2146, 'n_words': 388}