<commit_before># coding: utf8
from __future__ import unicode_literals

import sys
import unittest

PYTHON3 = sys.version_info >= (3,)
V8_0_0 = sys.version_info >= (3, 5)
V10_0_0 = sys.version_info >= (3, 7)


class SmokeTest(unittest.TestCase):

    def test_cafe(self):
        from pyuca import Collator
        c = Collator()

        self.assertEqual(
            sorted(["cafe", "caff", "café"]),
            ["cafe", "caff", "café"]
        )
        self.assertEqual(
            sorted(["cafe", "caff", "café"], key=c.sort_key),
            ["cafe", "café", "caff"]
        )


class UtilsTest(unittest.TestCase):

    def test_hexstrings2int(self):
        from pyuca.utils import hexstrings2int
        self.assertEqual(
            hexstrings2int(["0000", "0001", "FFFF"]),
            [0, 1, 65535]
        )

    def test_int2hexstrings(self):
        from pyuca.utils import int2hexstrings
        self.assertEqual(
            int2hexstrings([0, 1, 65535]),
            ["0000", "0001", "FFFF"]
        )

    def test_format_collation_elements(self):
        from pyuca.utils import format_collation_elements
        self.assertEqual(
            format_collation_elements([[1, 2, 3], [4, 5]]),
            "[0001.0002.0003], [0004.0005]"
        )

    def test_format_collation_elements_none(self):
        from pyuca.utils import format_collation_elements
        self.assertEqual(
            format_collation_elements(None),
            None
        )

    def test_format_sort_key(self):
        from pyuca.utils import format_sort_key
        self.assertEqual(
            format_sort_key([0, 1, 65535]),
            "| 0001 FFFF"
        )


class TrieTest(unittest.TestCase):

    def __init__(self, *args, **kwargs):
        from pyuca.trie import Trie
        super(TrieTest, self).__init__(*args, **kwargs)
        self.t = Trie()

    def test_1(self):
        self.t.add("foo", "bar")
        self.assertEqual(self.t.find_prefix("fo"), ("", None, "fo"))
        self.assertEqual(self.t.find_prefix("foo"), ("foo", "bar", ""))
        self.assertEqual(self.t.find_prefix("food"), ("foo", "bar", "d"))

    def test_2(self):
        self.t.add("a", "yes")
        self.t.add("abc", "yes")
        self.assertEqual(self.t.find_prefix("abdc"), ("a", "yes", "bdc"))


class FromFullTest(unittest.TestCase):

    def __init__(self, *args, **kwargs):
        from pyuca import Collator
        super(FromFullTest, self).__init__(*args, **kwargs)
        self.c = Collator()
        (0, 74, 33, 0, 2, 2, 0)

    @unittest.skipIf(not PYTHON3, "only matches Python 3's UCA version")
    def test_1(self):
        self.assertEqual(
            self.c.sort_key("\u0332\u0334"),
            (0x0000, 0x004A, 0x0021, 0x0000, 0x0002, 0x0002, 0x0000)
        )

    @unittest.skipIf(not PYTHON3, "only matches Python 3's UCA version")
    @unittest.skipIf(V10_0_0, "not for UCA version 10.0.0")
    def test_2(self):
        self.assertEqual(
            self.c.sort_key("\u0430\u0306\u0334"),
            (0x1991, 0x0000, 0x0020, 0x004A, 0x0000, 0x0002, 0x0002, 0x0000)
        )

    @unittest.skipIf(not PYTHON3, "only matches Python 3's UCA version")
    @unittest.skipIf(V10_0_0, "not for UCA version 10.0.0")
    def test_3(self):
        self.assertEqual(
            self.c.sort_key("\<KEY>"),
            (0x2571, 0x2587, 0x258A, 0x15EB, 0x0000, 0x0020, 0x0020, 0x0020,
                0x0020, 0x0000, 0x0002, 0x0002, 0x0002, 0x0002, 0x0000)
        )

    @unittest.skipIf(not PYTHON3, "only matches Python 3's UCA version")
    @unittest.skipIf(V10_0_0, "not for UCA version 10.0.0")
    def test_4(self):
        self.assertEqual(
            self.c.sort_key("\<KEY>"),
            (0xFB40, 0xCE00, 0x025D, 0x0000, 0x0020,
                0x0020, 0x0000, 0x0002, 0x0002, 0x0000)
        )

    @unittest.skipIf(not PYTHON3, "only matches Python 3's UCA version")
    @unittest.skipIf(V10_0_0, "not for UCA version 10.0.0")
    def test_5(self):
        self.assertEqual(
            self.c.sort_key("\u3400\u0021"),
            (0xFB80, 0xB400, 0x025D, 0x0000, 0x0020,
                0x0020, 0x0000, 0x0002, 0x0002, 0x0000)
        )
<commit_msg>exclude certain regression tests for 8.0.0 and 10.0.0 collators<commit_after># coding: utf8
from __future__ import unicode_literals

import sys
import unittest

PYTHON3 = sys.version_info >= (3,)
V8_0_0 = sys.version_info >= (3, 5)
V10_0_0 = sys.version_info >= (3, 7)


class SmokeTest(unittest.TestCase):

    def test_cafe(self):
        from pyuca import Collator
        c = Collator()

        self.assertEqual(
            sorted(["cafe", "caff", "café"]),
            ["cafe", "caff", "café"]
        )
        self.assertEqual(
            sorted(["cafe", "caff", "café"], key=c.sort_key),
            ["cafe", "café", "caff"]
        )


class UtilsTest(unittest.TestCase):

    def test_hexstrings2int(self):
        from pyuca.utils import hexstrings2int
        self.assertEqual(
            hexstrings2int(["0000", "0001", "FFFF"]),
            [0, 1, 65535]
        )

    def test_int2hexstrings(self):
        from pyuca.utils import int2hexstrings
        self.assertEqual(
            int2hexstrings([0, 1, 65535]),
            ["0000", "0001", "FFFF"]
        )

    def test_format_collation_elements(self):
        from pyuca.utils import format_collation_elements
        self.assertEqual(
            format_collation_elements([[1, 2, 3], [4, 5]]),
            "[0001.0002.0003], [0004.0005]"
        )

    def test_format_collation_elements_none(self):
        from pyuca.utils import format_collation_elements
        self.assertEqual(
            format_collation_elements(None),
            None
        )

    def test_format_sort_key(self):
        from pyuca.utils import format_sort_key
        self.assertEqual(
            format_sort_key([0, 1, 65535]),
            "| 0001 FFFF"
        )


class TrieTest(unittest.TestCase):

    def __init__(self, *args, **kwargs):
        from pyuca.trie import Trie
        super(TrieTest, self).__init__(*args, **kwargs)
        self.t = Trie()

    def test_1(self):
        self.t.add("foo", "bar")
        self.assertEqual(self.t.find_prefix("fo"), ("", None, "fo"))
        self.assertEqual(self.t.find_prefix("foo"), ("foo", "bar", ""))
        self.assertEqual(self.t.find_prefix("food"), ("foo", "bar", "d"))

    def test_2(self):
        self.t.add("a", "yes")
        self.t.add("abc", "yes")
        self.assertEqual(self.t.find_prefix("abdc"), ("a", "yes", "bdc"))


class FromFullTest(unittest.TestCase):

    def __init__(self, *args, **kwargs):
        from pyuca import Collator
        super(FromFullTest, self).__init__(*args, **kwargs)
        self.c = Collator()
        (0, 74, 33, 0, 2, 2, 0)

    @unittest.skipIf(not PYTHON3, "only matches Python 3's UCA version")
    def test_1(self):
        self.assertEqual(
            self.c.sort_key("\u0332\u0334"),
            (0x0000, 0x004A, 0x0021, 0x0000, 0x0002, 0x0002, 0x0000)
        )

    @unittest.skipIf(not PYTHON3, "only matches Python 3's UCA version")
    @unittest.skipIf(V8_0_0, "not for UCA version 8.0.0")
    @unittest.skipIf(V10_0_0, "not for UCA version 10.0.0")
    def test_2(self):
        self.assertEqual(
            self.c.sort_key("\<KEY>"),
            (0x1991, 0x0000, 0x0020, 0x004A, 0x0000, 0x0002, 0x0002, 0x0000)
        )

    @unittest.skipIf(not PYTHON3, "only matches Python 3's UCA version")
    @unittest.skipIf(V8_0_0, "not for UCA version 8.0.0")
    @unittest.skipIf(V10_0_0, "not for UCA version 10.0.0")
    def test_3(self):
        self.assertEqual(
            self.c.sort_key("\<KEY>"),
            (0x2571, 0x2587, 0x258A, 0x15EB, 0x0000, 0x0020, 0x0020, 0x0020,
                0x0020, 0x0000, 0x0002, 0x0002, 0x0002, 0x0002, 0x0000)
        )

    @unittest.skipIf(not PYTHON3, "only matches Python 3's UCA version")
    @unittest.skipIf(V8_0_0, "not for UCA version 8.0.0")
    @unittest.skipIf(V10_0_0, "not for UCA version 10.0.0")
    def test_4(self):
        self.assertEqual(
            self.c.sort_key("\<KEY>"),
            (0xFB40, 0xCE00, 0x025D, 0x0000, 0x0020,
                0x0020, 0x0000, 0x0002, 0x0002, 0x0000)
        )

    @unittest.skipIf(not PYTHON3, "only matches Python 3's UCA version")
    @unittest.skipIf(V8_0_0, "not for UCA version 8.0.0")
    @unittest.skipIf(V10_0_0, "not for UCA version 10.0.0")
    def test_5(self):
        self.assertEqual(
            self.c.sort_key("\<KEY>"),
            (0xFB80, 0xB400, 0x025D, 0x0000, 0x0020,
                0x0020, 0x0000, 0x0002, 0x0002, 0x0000)
        )

{'fraction_non_alphanumeric': 0.1397212543554007, 'fraction_numerical': 0.09605110336817654, 'mean_word_length': 2.4035573122529645, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 15, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1899034', 'n_tokens_mistral': 3681, 'n_tokens_neox': 3076, 'n_words': 642}