<commit_before>"""This module contains unit tests for the spider host's queues module."""

import datetime
import json
import unittest
import uuid

import mock

import clf.spider
from clf.spider_host import queues


class MyMessage(queues.SpiderHostMessage):

    @classmethod
    def get_schema(cls):
        schema = {
            "type": "object",
            "properties": {
                "uuid": {
                    "type": "string",
                    "minLength": 1,
                },
                "spider_name": {
                    "type": "string",
                    "minLength": 1,
                },
                "spider_args": {
                    "type": "array",
                    "items": {
                        "type": "string",
                        "minLength": 1,
                    },
                },
            },
            "required": [
                "uuid",
                "spider_name",
                "spider_args",
            ],
            "additionalProperties": False,
        }
        return schema


class TestSpiderHostQueue(unittest.TestCase):

    def test_write_message_encrypt_all_ok(self):
        mock_sqs_queue = mock.Mock()
        queue = queues.SpiderHostQueue(mock_sqs_queue)

        unencrypted_message = MyMessage(
            spider_name="dave",
            spider_args=[
                "dave",
                "was",
                "here",
            ],
        )

        mock_write_message_method = mock.Mock(return_value=unencrypted_message)
        name_of_method_to_patch = "clf.util.queues.Queue.write_message"
        with mock.patch(name_of_method_to_patch, mock_write_message_method):
            encrypted_message = queue.write_message(unencrypted_message)
            self.assertIsNotNone(encrypted_message)

            self.assertEqual(
                mock_write_message_method.call_args_list,
                [mock.call(queue, encrypted_message)])

    def test_write_message_that_is_none(self):
        mock_sqs_queue = mock.Mock()
        queue = queues.SpiderHostQueue(mock_sqs_queue)

        unencrypted_message = None

        mock_write_message_method = mock.Mock(return_value=unencrypted_message)
        name_of_method_to_patch = "clf.util.queues.Queue.write_message"
        with mock.patch(name_of_method_to_patch, mock_write_message_method):
            encrypted_message = queue.write_message(unencrypted_message)
            self.assertIsNone(encrypted_message)

            self.assertEqual(
                mock_write_message_method.call_args_list,
                [mock.call(queue, encrypted_message)])

    def test_read_message_decrypt_all_ok(self):
        mock_sqs_queue = mock.Mock()
        queue = queues.SpiderHostQueue(mock_sqs_queue)

        encrypted_message = MyMessage(
            spider_name="dave",
            spider_args=[
                "dave",
                "was",
                "here",
            ],
        )

        mock_read_message_method = mock.Mock(return_value=encrypted_message)
        name_of_method_to_patch = "clf.util.queues.Queue.read_message"
        with mock.patch(name_of_method_to_patch, mock_read_message_method):
            decrypted_message = queue.read_message()
            self.assertIsNotNone(decrypted_message)

        self.assertEqual(
            mock_read_message_method.call_args_list,
            [mock.call(queue)])

    def test_read_message_no_message_to_decrypt(self):
        mock_sqs_queue = mock.Mock()
        queue = queues.SpiderHostQueue(mock_sqs_queue)

        mock_read_message_method = mock.Mock(return_value=None)
        name_of_method_to_patch = "clf.util.queues.Queue.read_message"
        with mock.patch(name_of_method_to_patch, mock_read_message_method):
            decrypted_message = queue.read_message()
            self.assertIsNone(decrypted_message)

        self.assertEqual(
            mock_read_message_method.call_args_list,
            [mock.call(queue)])


class TestCrawlRequestQueue(unittest.TestCase):

    def test_get_message_class_returns_expected_message_class(self):
        self.assertEqual(
            queues.CrawlRequestQueue.get_message_class(),
            queues.CrawlRequest)

    def test_get_queue_name_prefix_returns_expected_prefix(self):
        self.assertEqual(
            queues.CrawlRequestQueue.get_queue_name_prefix(),
            "clf_creq_")


class TestCrawlRequest(unittest.TestCase):

    def test_process_spider_not_found(self):
        """Verify that when a spider can't be found we get
        a ```clf.spider.SC_SPIDER_NOT_FOUND``` response
        from ```queues.CrawlRequest.process()```."""
        message = queues.CrawlRequest(
            spider_name="dave",
            spider_args=[])
        local_spider_repo = mock.Mock()
        local_spider_repo.get_spider_class.return_value = None
        crawl_response = message.process(local_spider_repo)
        self.assertIsNotNone(crawl_response)
        cr = crawl_response.crawl_response
        self.assertIsNotNone(cr)
        self.assertEqual(cr.status_code, clf.spider.SC_SPIDER_NOT_FOUND)        
        self.assertEqual(
            local_spider_repo.get_spider_class.call_args_list,
            [mock.call(message.spider_name)])

    def test_process_all_good(self):
        """Verify the 'all good' scenario when calling
        ```queues.CrawlRequest.process()```."""
        crawl_request = queues.CrawlRequest(
            spider_name="dave",
            spider_args=[1, 2, 3])

        spider_walk_return_value_mock = mock.Mock()
        spider_mock = mock.Mock()
        spider_mock.walk.return_value = spider_walk_return_value_mock

        spider_class_mock = mock.Mock(return_value=spider_mock)

        local_spider_repo = mock.Mock()
        local_spider_repo.get_spider_class.return_value = spider_class_mock

        crawl_response = crawl_request.process(local_spider_repo)

        self.assertIsNotNone(crawl_response)

        # crawl response should have same basic parameters as
        # as crawl request
        self.assertEqual(crawl_request.uuid, crawl_response.uuid)
        self.assertEqual(crawl_request.spider_name, crawl_response.spider_name)
        self.assertEqual(crawl_request.spider_args, crawl_response.spider_args)

        # crawl response should have a metrics attribute that's<commit_msg>add spider host test 4 spider ctr raise exception<commit_after>"""This module contains unit tests for the spider host's queues module."""

import datetime
import json
import unittest
import uuid

import mock

import clf.spider
from clf.spider_host import queues


class MyMessage(queues.SpiderHostMessage):

    @classmethod
    def get_schema(cls):
        schema = {
            "type": "object",
            "properties": {
                "uuid": {
                    "type": "string",
                    "minLength": 1,
                },
                "spider_name": {
                    "type": "string",
                    "minLength": 1,
                },
                "spider_args": {
                    "type": "array",
                    "items": {
                        "type": "string",
                        "minLength": 1,
                    },
                },
            },
            "required": [
                "uuid",
                "spider_name",
                "spider_args",
            ],
            "additionalProperties": False,
        }
        return schema


class TestSpiderHostQueue(unittest.TestCase):

    def test_write_message_encrypt_all_ok(self):
        mock_sqs_queue = mock.Mock()
        queue = queues.SpiderHostQueue(mock_sqs_queue)

        unencrypted_message = MyMessage(
            spider_name="dave",
            spider_args=[
                "dave",
                "was",
                "here",
            ],
        )

        mock_write_message_method = mock.Mock(return_value=unencrypted_message)
        name_of_method_to_patch = "clf.util.queues.Queue.write_message"
        with mock.patch(name_of_method_to_patch, mock_write_message_method):
            encrypted_message = queue.write_message(unencrypted_message)
            self.assertIsNotNone(encrypted_message)

            self.assertEqual(
                mock_write_message_method.call_args_list,
                [mock.call(queue, encrypted_message)])

    def test_write_message_that_is_none(self):
        mock_sqs_queue = mock.Mock()
        queue = queues.SpiderHostQueue(mock_sqs_queue)

        unencrypted_message = None

        mock_write_message_method = mock.Mock(return_value=unencrypted_message)
        name_of_method_to_patch = "clf.util.queues.Queue.write_message"
        with mock.patch(name_of_method_to_patch, mock_write_message_method):
            encrypted_message = queue.write_message(unencrypted_message)
            self.assertIsNone(encrypted_message)

            self.assertEqual(
                mock_write_message_method.call_args_list,
                [mock.call(queue, encrypted_message)])

    def test_read_message_decrypt_all_ok(self):
        mock_sqs_queue = mock.Mock()
        queue = queues.SpiderHostQueue(mock_sqs_queue)

        encrypted_message = MyMessage(
            spider_name="dave",
            spider_args=[
                "dave",
                "was",
                "here",
            ],
        )

        mock_read_message_method = mock.Mock(return_value=encrypted_message)
        name_of_method_to_patch = "clf.util.queues.Queue.read_message"
        with mock.patch(name_of_method_to_patch, mock_read_message_method):
            decrypted_message = queue.read_message()
            self.assertIsNotNone(decrypted_message)

        self.assertEqual(
            mock_read_message_method.call_args_list,
            [mock.call(queue)])

    def test_read_message_no_message_to_decrypt(self):
        mock_sqs_queue = mock.Mock()
        queue = queues.SpiderHostQueue(mock_sqs_queue)

        mock_read_message_method = mock.Mock(return_value=None)
        name_of_method_to_patch = "clf.util.queues.Queue.read_message"
        with mock.patch(name_of_method_to_patch, mock_read_message_method):
            decrypted_message = queue.read_message()
            self.assertIsNone(decrypted_message)

        self.assertEqual(
            mock_read_message_method.call_args_list,
            [mock.call(queue)])


class TestCrawlRequestQueue(unittest.TestCase):

    def test_get_message_class_returns_expected_message_class(self):
        self.assertEqual(
            queues.CrawlRequestQueue.get_message_class(),
            queues.CrawlRequest)

    def test_get_queue_name_prefix_returns_expected_prefix(self):
        self.assertEqual(
            queues.CrawlRequestQueue.get_queue_name_prefix(),
            "clf_creq_")


class TestCrawlRequest(unittest.TestCase):

    def test_process_spider_not_found(self):
        """Verify that when a spider can't be found we get
        a ```clf.spider.SC_SPIDER_NOT_FOUND``` response
        from ```queues.CrawlRequest.process()```."""
        message = queues.CrawlRequest(
            spider_name="dave",
            spider_args=[])
        local_spider_repo = mock.Mock()
        local_spider_repo.get_spider_class.return_value = None
        crawl_response = message.process(local_spider_repo)
        self.assertIsNotNone(crawl_response)
        cr = crawl_response.crawl_response
        self.assertIsNotNone(cr)
        self.assertEqual(cr.status_code, clf.spider.SC_SPIDER_NOT_FOUND)        
        self.assertEqual(
            local_spider_repo.get_spider_class.call_args_list,
            [mock.call(message.spider_name)])

    def test_process_exception_thrown_by_spider_ctr(self):
        """When a spider's constructure raises an exception, verify
        ```queues.CrawlRequest.process()``` correct catches the exception
        and returns an appropriate crawl response."""
        crawl_request = queues.CrawlRequest(
            spider_name="dave",
            spider_args=[1, 2, 3])

        spider_class_mock = mock.Mock(side_effect=Exception)

        local_spider_repo = mock.Mock()
        local_spider_repo.get_spider_class.return_value = spider_class_mock

        crawl_response = crawl_request.process(local_spider_repo)

    def test_process_all_good(self):
        """Verify the 'all good' scenario when calling
        ```queues.CrawlRequest.process()```."""
        crawl_request = queues.CrawlRequest(
            spider_name="dave",
            spider_args=[1, 2, 3])

        spider_walk_return_value_mock = mock.Mock()
        spider_mock = mock.Mock()
        spider_mock.walk.return_value = spider_walk_return_value_mock

        spider_class_mock = mock.Mock(return_value=spider_mock)

        local_spider_repo = mock.Mock()
        local_spider_repo.get_spider_class.return_value = spider_class_mock

        crawl_response = crawl_request.process(local_spider_repo)

        self.assertIsNotNone(crawl_response)

        # crawl response should have same basic parameters as
        # as crawl request
        self.assertEqual(crawl_request.uuid, crawl_response.uuid)
        self.assertEqual(crawl_request.spider_name, crawl_response.spider_name)
        self.assertEqual(crawl_request.spider_args, crawl_response.spider_args)

        # crawl response should have a metrics attribute that's
{'fraction_non_alphanumeric': 0.08476989956528257, 'fraction_numerical': 0.0011992205066706642, 'mean_word_length': 2.324115595416044, 'pattern_counts': {'":': 30, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '812463', 'n_tokens_mistral': 4083, 'n_tokens_neox': 3875, 'n_words': 570}