<commit_before>from unittest import TestCase

from framewirc import exceptions
from framewirc.utils import chunk_message, RequiredAttributesMixin, to_bytes, to_unicode


class TestToUnicode(TestCase):
    def test_already_unicode(self):
        text = 'тнιѕ ιѕ αℓяєα∂у υηι¢σ∂є'
        result = to_unicode(text)
        self.assertEqual(result, text)

    def test_ascii(self):
        text = b'This is just plain ASCII'
        expected = 'This is just plain ASCII'
        result = to_unicode(text)
        self.assertEqual(result, expected)

    def test_latin_1(self):
        text = b'Ume\xe5'
        expected = 'Umeå'
        result = to_unicode(text)
        self.assertEqual(result, expected)

    def test_windows_1250(self):
        text = b'Miko\xb3<NAME>'
        expected = 'Miko<NAME>'
        result = to_unicode(text)
        self.assertEqual(result, expected)

    def test_not_bytes_or_string(self):
        with self.assertRaises(AttributeError):
            to_unicode(None)


class TestToBytes(TestCase):
    def test_unicode(self):
        text = 'ಠ_ಠ'
        expected = b'\xe0\xb2\xa0_\xe0\xb2\xa0'
        result = to_bytes(text)
        self.assertEqual(result, expected)

    def test_already_bytes(self):
        text = b'bytes!'
        result = to_bytes(text)
        self.assertEqual(result, text)

    def test_not_bytes_or_string(self):
        with self.assertRaises(AttributeError):
            to_bytes(None)

<commit_msg>Test for decoding utf8 encoded bytes<commit_after>from unittest import TestCase

from framewirc import exceptions
from framewirc.utils import chunk_message, RequiredAttributesMixin, to_bytes, to_unicode


class TestToUnicode(TestCase):
    def test_already_unicode(self):
        text = 'тнιѕ ιѕ αℓяєα∂у υηι¢σ∂є'
        result = to_unicode(text)
        self.assertEqual(result, text)

    def test_ascii(self):
        text = b'This is just plain ASCII'
        expected = 'This is just plain ASCII'
        result = to_unicode(text)
        self.assertEqual(result, expected)

    def test_latin_1(self):
        text = b'Ume\xe5'
        expected = 'Umeå'
        result = to_unicode(text)
        self.assertEqual(result, expected)

    def test_utf8(self):
        text = b"Rhoi'r ffidil yn y t\xc3\xb4"
        expected = "Rhoi'r ffidil yn y tô"
        result = to_unicode(text)
        self.assertEqual(result, expected)

    def test_windows_1250(self):
        text = b'Miko\xb3<NAME>'
        expected = '<NAME>'
        result = to_unicode(text)
        self.assertEqual(result, expected)

    def test_not_bytes_or_string(self):
        with self.assertRaises(AttributeError):
            to_unicode(None)


class TestToBytes(TestCase):
    def test_unicode(self):
        text = 'ಠ_ಠ'
        expected = b'\xe0\xb2\xa0_\xe0\xb2\xa0'
        result = to_bytes(text)
        self.assertEqual(result, expected)

    def test_already_bytes(self):
        text = b'bytes!'
        result = to_bytes(text)
        self.assertEqual(result, text)

    def test_not_bytes_or_string(self):
        with self.assertRaises(AttributeError):
            to_bytes(None)


{'fraction_non_alphanumeric': 0.0939297124600639, 'fraction_numerical': 0.009584664536741214, 'mean_word_length': 2.705325443786982, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6137132', 'n_tokens_mistral': 1053, 'n_tokens_neox': 1004, 'n_words': 215}