<commit_before>__author__ = 'mengpeng'
from unittest import TestCase
from pycrawler.scraper import Scraper
from pycrawler.scraper import DefaultScraper
from pycrawler.exception import ScraperException


class TestScraper(TestCase):
    def test_register(self):
        self.assertRaises(ScraperException, Scraper.register, 0)
        self.assertEqual(DefaultScraper, Scraper.Dict['DefaultScraper'])

    def test_getScraper(self):
        self.assertRaises(ScraperException, Scraper.getScraper, 'None')
        self.assertEqual(DefaultScraper, Scraper.getScraper('DefaultScraper'))
        self.assertIsInstance(Scraper.getScraper('DefaultScraper')(SpiderTest('testspider')), DefaultScraper)


class TestDefaultScraper(TestCase):
    def test_fetchone(self):
        sp = SpiderTest('testspider')
        s = DefaultScraper(sp)
        _, html = s.fetchone('http://www.google.com')
        self.assertNotEqual(0, len(html))

    def test_fetch(self):
        sp = SpiderTest('testspider')
        s = DefaultScraper(sp)
        results = s.fetch(['http://www.google.com',
                           'http://www.baidu.com',
                           'http://www.zhihu.com'])
        self.assertEqual(3, len(results))
        for value in results.itervalues():
            self.assertNotEqual(0, len(value))

    def test_url(self):
        url = 'http://www.google.com'
        data = {'a': 1, 'b': 2, 'c': 3}
        full = DefaultScraper.encodeurl('POST', url, data)
        self.assertTrue('<args>' in full)
        url2, data2 = DefaultScraper.parseurl(full)
        self.assertEqual(url, url2)
        self.assertEqual(data, data2)


class SpiderTest:
    def __init__(self, name):
        self.name = name

    def addtask(self, task):
        print('{0} got url(s):'.format(self.name))
        print(task)<commit_msg>update test cases<commit_after>__author__ = 'mengpeng'
from unittest import TestCase
from pycrawler.scraper import Scraper
from pycrawler.scraper import DefaultScraper
from pycrawler.exception import ScraperException
from pycrawler.logger import Logger


class TestScraper(TestCase):
    def test_register(self):
        self.assertRaises(ScraperException, Scraper.register, 0)
        self.assertEqual(DefaultScraper, Scraper.Dict['DefaultScraper'])

    def test_getScraper(self):
        self.assertRaises(ScraperException, Scraper.getScraper, 'None')
        self.assertEqual(DefaultScraper, Scraper.getScraper('DefaultScraper'))
        self.assertIsInstance(Scraper.getScraper('DefaultScraper')(SpiderTest('testspider')), DefaultScraper)


class TestDefaultScraper(TestCase):
    def test_fetchone(self):
        sp = SpiderTest('testspider')
        s = DefaultScraper(sp)
        _, html = s.fetchone('http://www.google.com')
        self.assertNotEqual(0, len(html))

    def test_fetch(self):
        sp = SpiderTest('testspider')
        s = DefaultScraper(sp)
        results = s.fetch(['http://www.google.com',
                           'http://www.baidu.com',
                           'http://www.zhihu.com'])
        self.assertEqual(3, len(results))
        for value in results.itervalues():
            self.assertNotEqual(0, len(value))

    def test_url(self):
        url = 'http://www.google.com'
        data = {'a': 1, 'b': 2, 'c': 3}
        full = DefaultScraper.encodeurl('POST', url, data)
        self.assertTrue('<args>' in full)
        url2, data2 = DefaultScraper.parseurl(full)
        self.assertEqual(url, url2)
        self.assertEqual(data, data2)


class SpiderTest:
    def __init__(self, name):
        self.name = name
        self.logger = Logger('Default')

    def addtask(self, task):
        print('{0} got url(s):'.format(self.name))
        print(task)
{'fraction_non_alphanumeric': 0.1263412017167382, 'fraction_numerical': 0.006437768240343348, 'mean_word_length': 2.967021276595745, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 10, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4699256', 'n_tokens_mistral': 1219, 'n_tokens_neox': 1191, 'n_words': 217}