<commit_before># -*- coding: utf-8 -*-
from .base_case import ChatBotTestCase


class CleanFuncTests(ChatBotTestCase):

    def setUp(self):
        super(CleanFuncTests, self).setUp()

        from chatterbot.apis import clean
        self.clean = clean

    def test_linebreaks_replaced_with_spaces(self):

        text = '\tThe quick \nbrown fox \rjumps over \vthe \alazy \fdog\\.'
        clean_text = self.clean(text)
        normal_text = 'The quick brown fox jumps over \vthe \alazy \fdog\\.'

        self.assertEqual(clean_text, normal_text)

    def test_leeding_or_trailing_whitespace_removed(self):

        text = '     The quick brown fox jumps over the lazy dog.   '
        clean_text = self.clean(text)
        normal_text = 'The quick brown fox jumps over the lazy dog.'

        self.assertEqual(clean_text, normal_text)

    def test_consecutive_spaces_removed(self):

        text = 'The       quick brown     fox      jumps over the lazy dog.'
        clean_text = self.clean(text)
        normal_text = 'The quick brown fox jumps over the lazy dog.'

        self.assertEqual(clean_text, normal_text)

    def test_html_characters_restored(self):

        # implicit concat
        text = 'The quick brown fox &lt;b&gt;jumps&lt;/b&gt; over'
        ' the <a href="http://lazy.com">lazy</a> dog.'

        normal_text = 'The quick brown fox <b>jumps</b> over'
        ' the <a href="http://lazy.com">lazy</a> dog.'

        clean_text = self.clean(text)

        self.assertEqual(clean_text, normal_text)

    def test_non_ascii_chars_removed(self):

        text = u"Klüft skräms inför på fédéral électoral große"
        clean_text = self.clean(text)
        normal_text = "Kluft skrams infor pa federal electoral groe"

        self.assertEqual(clean_text, normal_text)
<commit_msg>Fix methods names.<commit_after># -*- coding: utf-8 -*-
from .base_case import ChatBotTestCase


class CleanFuncTests(ChatBotTestCase):

    def setUp(self):
        super(CleanFuncTests, self).setUp()

        from chatterbot.apis import clean
        self.clean = clean

    def test_linebreaks_removed(self):

        text = '\tThe quick \nbrown fox \rjumps over \vthe \alazy \fdog\\.'
        clean_text = self.clean(text)
        normal_text = 'The quick brown fox jumps over \vthe \alazy \fdog\\.'

        self.assertEqual(clean_text, normal_text)

    def test_leading_or_trailing_whitespace_removed(self):

        text = '     The quick brown fox jumps over the lazy dog.   '
        clean_text = self.clean(text)
        normal_text = 'The quick brown fox jumps over the lazy dog.'

        self.assertEqual(clean_text, normal_text)

    def test_consecutive_spaces_removed(self):

        text = 'The       quick brown     fox      jumps over the lazy dog.'
        clean_text = self.clean(text)
        normal_text = 'The quick brown fox jumps over the lazy dog.'

        self.assertEqual(clean_text, normal_text)

    def test_html_characters_restored(self):

        # implicit concat
        text = 'The quick brown fox &lt;b&gt;jumps&lt;/b&gt; over'
        ' the <a href="http://lazy.com">lazy</a> dog.'

        normal_text = 'The quick brown fox <b>jumps</b> over'
        ' the <a href="http://lazy.com">lazy</a> dog.'

        clean_text = self.clean(text)

        self.assertEqual(clean_text, normal_text)

    def test_non_ascii_chars_removed(self):

        text = u"Klüft skräms inför på fédéral électoral große"
        clean_text = self.clean(text)
        normal_text = "Kluft skrams infor pa federal electoral groe"

        self.assertEqual(clean_text, normal_text)

{'fraction_non_alphanumeric': 0.09844703272323904, 'fraction_numerical': 0.0005546311702717693, 'mean_word_length': 2.6214859437751006, 'pattern_counts': {'":': 0, '<': 15, '<?xml version=': 0, '>': 15, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5352074', 'n_tokens_mistral': 1183, 'n_tokens_neox': 1113, 'n_words': 300}