<commit_before># -*- coding: utf-8 -*-

import unittest

import tldextract
from validators import ipv6

from spam_lists.clients import spamhaus_zen, spamhaus_zen_classification,\
spamhaus_dbl, spamhaus_dbl_classification, surbl_multi,\
surbl_multi_classification
from spam_lists.structures import AddressListItem

def ip_or_registered_domain(host):
    registered_domain = tldextract.extract(host).registered_domain
    return host if not registered_domain else registered_domain


def url_from_host(host):
    if ipv6(host):
        host = '['+host+']'
    return u'http://'+host


def get_expected_classification(classification, return_codes):
    return set(v for k, v in classification.items()
               if k in return_codes)
    

class ClientTest(object):
    @classmethod
    def setUpClass(cls):
        cls.listed_url = url_from_host(cls.listed)
        cls.not_listed_url = url_from_host(cls.not_listed)
        cls.urls_with_listed = cls.not_listed_url, cls.listed_url
        cls.urls_without_listed = cls.not_listed_url, url_from_host(
                                                                    cls.not_listed_2
                                                                    )
        
        cls.listed_item = AddressListItem(
                                  ip_or_registered_domain(cls.listed),
                                  cls.tested_client,
                                  cls.classification
                                  )
    
    def test__contains__for_not_listed(self):
        actual = self.not_listed in self.tested_client
        self.assertFalse(actual)
        
    def test_contains_for_listed(self):
        actual = self.listed in self.tested_client
        self.assertTrue(actual)
        
    def test_lookup_for_not_listed(self):
        actual = self.tested_client.lookup(self.not_listed)
        self.assertIsNone(actual)
        
    def test_lookup_for_listed(self):
        actual = self.tested_client.lookup(self.listed)
        self.assertEqual(self.listed_item, actual)
        
    def test_any_match_for_not_listed(self):
        actual = self.tested_client.any_match(self.urls_without_listed)
        self.assertFalse(actual)    
    
    def test_any_match_for_listed(self):
        actual = self.tested_client.any_match(self.urls_with_listed)
        self.assertTrue(actual)
        
    def test_filter_matching_for_not_listed(self):
        actual = list(self.tested_client.filter_matching(self.urls_without_listed))
        self.assertItemsEqual([], actual)
        
    def test_filter_matching_for_listed(self):
        expected = [self.listed_url]
        actual = list(self.tested_client.filter_matching(self.urls_with_listed))
        self.assertItemsEqual(expected, actual)
        
    def test_lookup_matching_for_not_listed(self):
        actual = list(self.tested_client.lookup_matching(self.urls_without_listed))
        self.assertItemsEqual([], actual)
        
    def test_lookup_matching_for_listed(self):
        expected = [self.listed_item]
        actual = list(self.tested_client.lookup_matching(self.urls_with_listed))
        self.assertItemsEqual(expected, actual)


reason_to_skip = (
                  'These tests are expected to fail frequently for users of'
                  ' public DNS resolvers:'
                  ' https://www.spamhaus.org/faq/section/DNSBL%20Usage#261'
                  )


@unittest.skip(reason_to_skip)
class SpamhausZenTest(ClientTest, unittest.TestCase):
    tested_client = spamhaus_zen
    listed = u'127.0.0.2'
    not_listed = u'127.0.0.1'
    not_listed_2 = u'8.8.8.8'
    classification = get_expected_classification(
                                                 spamhaus_zen_classification,
                                                 [2, 4, 10]
                                                 )
    

@unittest.skip(reason_to_skip)
class SpamhausDBLTest(ClientTest, unittest.TestCase):
    tested_client = spamhaus_dbl
    listed = 'dbltest.com'
    not_listed = 'example.com'
    not_listed_2 = 'google.com'
    classification = get_expected_classification(
                                                 spamhaus_dbl_classification,
                                                 [2]
                                                 )


expected_surbl_classification = get_expected_classification(
                                                            surbl_multi_classification,
                                                            [2, 126]
                                                            )

class SURBLTest(ClientTest):
    tested_client = surbl_multi
    classification = get_expected_classification(
                                                 surbl_multi_classification,
                                                 [2, 4, 8, 16, 32, 64]
                                                 )

class SURBLMultiIPTest(SURBLTest, unittest.TestCase):
    listed = u'127.0.0.2'
    not_listed = u'127.0.0.1'
    not_listed_2 = u'8.8.8.8'
    
class SURBLMultiDomainTest(SURBLTest, unittest.TestCase):
    listed = 'surbl-org-permanent-test-point.com'
    not_listed = 'test.com'
    not_listed_2 = 'google.com'


if __name__ == "__main__":
    #import sys;sys.argv = ['', 'Test.testName']
    unittest.main()<commit_msg>Add tests for hpHosts client<commit_after># -*- coding: utf-8 -*-

import unittest

import tldextract
from validators import ipv6

from spam_lists.clients import spamhaus_zen, spamhaus_zen_classification,\
spamhaus_dbl, spamhaus_dbl_classification, surbl_multi,\
surbl_multi_classification
from spam_lists.structures import AddressListItem
from spam_lists.service_models import HpHosts

def ip_or_registered_domain(host):
    registered_domain = tldextract.extract(host).registered_domain
    return host if not registered_domain else registered_domain


def url_from_host(host):
    if ipv6(host):
        host = '['+host+']'
    return u'http://'+host


def get_expected_classification(classification, return_codes):
    return set(v for k, v in classification.items()
               if k in return_codes)
    

class ClientTest(object):
    @classmethod
    def setUpClass(cls):
        cls.listed_url = url_from_host(cls.listed)
        cls.not_listed_url = url_from_host(cls.not_listed)
        cls.urls_with_listed = cls.not_listed_url, cls.listed_url
        cls.urls_without_listed = cls.not_listed_url, url_from_host(
                                                                    cls.not_listed_2
                                                                    )
        
        cls.listed_item = AddressListItem(
                                  ip_or_registered_domain(cls.listed),
                                  cls.tested_client,
                                  cls.classification
                                  )
    
    def test__contains__for_not_listed(self):
        actual = self.not_listed in self.tested_client
        self.assertFalse(actual)
        
    def test_contains_for_listed(self):
        actual = self.listed in self.tested_client
        self.assertTrue(actual)
        
    def test_lookup_for_not_listed(self):
        actual = self.tested_client.lookup(self.not_listed)
        self.assertIsNone(actual)
        
    def test_lookup_for_listed(self):
        actual = self.tested_client.lookup(self.listed)
        self.assertEqual(self.listed_item, actual)
        
    def test_any_match_for_not_listed(self):
        actual = self.tested_client.any_match(self.urls_without_listed)
        self.assertFalse(actual)    
    
    def test_any_match_for_listed(self):
        actual = self.tested_client.any_match(self.urls_with_listed)
        self.assertTrue(actual)
        
    def test_filter_matching_for_not_listed(self):
        actual = list(self.tested_client.filter_matching(self.urls_without_listed))
        self.assertItemsEqual([], actual)
        
    def test_filter_matching_for_listed(self):
        expected = [self.listed_url]
        actual = list(self.tested_client.filter_matching(self.urls_with_listed))
        self.assertItemsEqual(expected, actual)
        
    def test_lookup_matching_for_not_listed(self):
        actual = list(self.tested_client.lookup_matching(self.urls_without_listed))
        self.assertItemsEqual([], actual)
        
    def test_lookup_matching_for_listed(self):
        expected = [self.listed_item]
        actual = list(self.tested_client.lookup_matching(self.urls_with_listed))
        self.assertItemsEqual(expected, actual)


reason_to_skip = (
                  'These tests are expected to fail frequently for users of'
                  ' public DNS resolvers:'
                  ' https://www.spamhaus.org/faq/section/DNSBL%20Usage#261'
                  )


@unittest.skip(reason_to_skip)
class SpamhausZenTest(ClientTest, unittest.TestCase):
    tested_client = spamhaus_zen
    listed = u'127.0.0.2'
    not_listed = u'127.0.0.1'
    not_listed_2 = u'8.8.8.8'
    classification = get_expected_classification(
                                                 spamhaus_zen_classification,
                                                 [2, 4, 10]
                                                 )
    

@unittest.skip(reason_to_skip)
class SpamhausDBLTest(ClientTest, unittest.TestCase):
    tested_client = spamhaus_dbl
    listed = 'dbltest.com'
    not_listed = 'example.com'
    not_listed_2 = 'google.com'
    classification = get_expected_classification(
                                                 spamhaus_dbl_classification,
                                                 [2]
                                                 )


expected_surbl_classification = get_expected_classification(
                                                            surbl_multi_classification,
                                                            [2, 126]
                                                            )

class SURBLTest(ClientTest):
    tested_client = surbl_multi
    classification = get_expected_classification(
                                                 surbl_multi_classification,
                                                 [2, 4, 8, 16, 32, 64]
                                                 )

class SURBLMultiIPTest(SURBLTest, unittest.TestCase):
    listed = u'127.0.0.2'
    not_listed = u'127.0.0.1'
    not_listed_2 = u'8.8.8.8'
    
class SURBLMultiDomainTest(SURBLTest, unittest.TestCase):
    listed = 'surbl-org-permanent-test-point.com'
    not_listed = 'test.com'
    not_listed_2 = 'google.com'
    
hp_hosts = HpHosts('spam-lists-test-suite')

class HpHostsIPTest(ClientTest, unittest.TestCase):
    listed = u'172.16.17.32'
    not_listed = u'172.16.31.10'
    not_listed_2 = u'fd00:a516:7c1b:17cd:6d81:2137:bd2a:2c5b'
    tested_client = hp_hosts
    classification = set()
    
class HpHostsDomainTest(ClientTest, unittest.TestCase):
    listed = 'ecardmountain.com'
    not_listed = 'google.com'
    not_listed_2 = 'microsoft.com'
    tested_client = hp_hosts
    classification = set(['EMD'])


if __name__ == "__main__":
    #import sys;sys.argv = ['', 'Test.testName']
    unittest.main()
{'fraction_non_alphanumeric': 0.07905314872711032, 'fraction_numerical': 0.014738722644037517, 'mean_word_length': 1.888544891640867, 'pattern_counts': {'":': 2, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 2, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 14, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 62, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3081551', 'n_tokens_mistral': 3269, 'n_tokens_neox': 3094, 'n_words': 501}