<commit_before>"""This module contains unit tests for the ```spider``` module."""

import os
import sys
import unittest
import uuid

sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), "..")))

import spider

class TestCrawlResponse(unittest.TestCase):

    def test_ctr_with_just_status_code_ok(self):
        status_code = 42
        cr = spider.CrawlResponse(status_code)
        self.assertEqual(cr.status_code, status_code)
        self.assertIsNone(cr.data)
        self.assertIsNone(cr.status)

    def test_ctr_with_status_code_and_status_ok(self):
        status_code = 42
        status = str(uuid.uuid4())
        cr = spider.CrawlResponse(status_code, status=status)
        self.assertEqual(cr.status_code, status_code)
        self.assertIsNone(cr.data)
        self.assertEqual(cr.status, status)

    def test_ctr_with_status_code_status_and_data_ok(self):
        status_code = 42
        status = str(uuid.uuid4())
        data = {
            'data1': str(uuid.uuid4()),
            'data2': str(uuid.uuid4()),
        }
        cr = spider.CrawlResponse(status_code, data, status)
        self.assertEqual(cr.status_code, status_code)
        self.assertEqual(cr.data, data)
        self.assertEqual(cr.status, status)


class SpiderWithNoCrawlMethod(spider.Spider):
    pass


class SpiderWithCrawlMethodThatThrowsException(spider.Spider):

    def crawl(self):
        raise Exception()


class SpiderWithCrawlMethodThatReturnsNone(spider.Spider):

    def crawl(self):
        return None


class TestSpider(unittest.TestCase):

    def test_spider_ctr_sets_url_property(self):
        url = "http://www.example.com"
        my_spider = SpiderWithNoCrawlMethod("http://www.example.com")
        self.assertIsNotNone(my_spider.url)
        self.assertEqual(my_spider.url, url)

    def test_spider_with_no_crawl_method(self):
        url = "http://www.example.com"
        my_spider = SpiderWithNoCrawlMethod(url)
        rv = my_spider.walk()
        self.assertIsNotNone(rv)
        self.assertEqual(type(rv), spider.CrawlResponse)
        self.assertEqual(rv.status_code, spider.SC_CRAWL_NOT_IMPLEMENTED)

    def test_spider_with_crawl_method_that_raises_exception(self):
        url = "http://www.example.com"
        my_spider = SpiderWithCrawlMethodThatThrowsException(url)
        rv = my_spider.walk()
        self.assertIsNotNone(rv)
        self.assertEqual(type(rv), spider.CrawlResponse)
        self.assertEqual(rv.status_code, spider.SC_CRAWL_THREW_EXCEPTION)

    def test_spider_with_crawl_method_with_invalid_return_type(self):
        url = "http://www.example.com"
        my_spider = SpiderWithCrawlMethodThatReturnsNone(url)
        rv = my_spider.walk()
        self.assertIsNotNone(rv)
        self.assertEqual(type(rv), spider.CrawlResponse)
        self.assertEqual(rv.status_code, spider.SC_INVALID_CRAWL_RETURN_TYPE)<commit_msg>adding unit test for clf/spider.py to validate correct arg passing and return types for a spider's crawl method<commit_after>"""This module contains unit tests for the ```spider``` module."""

import os
import sys
import unittest
import uuid

sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), "..")))

import spider

class TestCrawlResponse(unittest.TestCase):

    def test_ctr_with_just_status_code_ok(self):
        status_code = 42
        cr = spider.CrawlResponse(status_code)
        self.assertEqual(cr.status_code, status_code)
        self.assertIsNone(cr.data)
        self.assertIsNone(cr.status)

    def test_ctr_with_status_code_and_status_ok(self):
        status_code = 42
        status = str(uuid.uuid4())
        cr = spider.CrawlResponse(status_code, status=status)
        self.assertEqual(cr.status_code, status_code)
        self.assertIsNone(cr.data)
        self.assertEqual(cr.status, status)

    def test_ctr_with_status_code_status_and_data_ok(self):
        status_code = 42
        status = str(uuid.uuid4())
        data = {
            'data1': str(uuid.uuid4()),
            'data2': str(uuid.uuid4()),
        }
        cr = spider.CrawlResponse(status_code, data, status)
        self.assertEqual(cr.status_code, status_code)
        self.assertEqual(cr.data, data)
        self.assertEqual(cr.status, status)


class TestSpider(unittest.TestCase):

    def test_spider_ctr_sets_url_property(self):
        class MySpider(spider.Spider):
            pass
        url = "http://www.example.com"
        my_spider = MySpider("http://www.example.com")
        self.assertIsNotNone(my_spider.url)
        self.assertEqual(my_spider.url, url)

    def test_spider_with_no_crawl_method(self):
        class MySpider(spider.Spider):
            pass
        url = "http://www.example.com"
        my_spider = MySpider(url)
        rv = my_spider.walk()
        self.assertIsNotNone(rv)
        self.assertEqual(type(rv), spider.CrawlResponse)
        self.assertEqual(rv.status_code, spider.SC_CRAWL_NOT_IMPLEMENTED)

    def test_spider_with_crawl_method_that_raises_exception(self):
        class MySpider(spider.Spider):
            def crawl(self):
                raise Exception()
        url = "http://www.example.com"
        my_spider = MySpider(url)
        rv = my_spider.walk()
        self.assertIsNotNone(rv)
        self.assertEqual(type(rv), spider.CrawlResponse)
        self.assertEqual(rv.status_code, spider.SC_CRAWL_THREW_EXCEPTION)

    def test_spider_with_crawl_method_with_invalid_return_type(self):
        class MySpider(spider.Spider):
            def crawl(self):
                return None
        url = "http://www.example.com"
        my_spider = MySpider(url)
        rv = my_spider.walk()
        self.assertIsNotNone(rv)
        self.assertEqual(type(rv), spider.CrawlResponse)
        self.assertEqual(rv.status_code, spider.SC_INVALID_CRAWL_RETURN_TYPE)

    def test_spider_correctly_passes_crawl_args_and_returns(self):
        my_arg1 = str(uuid.uuid4())
        my_arg2 = str(uuid.uuid4())
        my_crawl_response = spider.CrawlResponse(
            spider.SC_OK,
            {
                'data1': str(uuid.uuid4()),
                'data2': str(uuid.uuid4()),
            },
            "all ok!!!"
        )
        class MySpider(spider.Spider):
            def crawl(the_spider_self, arg1, arg2):
                self.assertEqual(arg1, my_arg1)
                self.assertEqual(arg2, my_arg2)
                return my_crawl_response
        url = "http://www.example.com"
        my_spider = MySpider(url)
        rv = my_spider.walk(my_arg1, my_arg2)
        self.assertIsNotNone(rv)
        self.assertEqual(type(rv), spider.CrawlResponse)
        self.assertEqual(rv, my_crawl_response)
{'fraction_non_alphanumeric': 0.10494103597551874, 'fraction_numerical': 0.006269592476489028, 'mean_word_length': 2.8527889591719378, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 11, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3687929', 'n_tokens_mistral': 2257, 'n_tokens_neox': 2146, 'n_words': 324}