<commit_before>from wordsalad.text import splitWestern
import unittest

class TestTokenisation(unittest.TestCase):
    
    def test_splitWestern_punctuation_treated_like_one_word(self):
        txt = "abc. def.,"

        res = list(splitWestern(txt))

        self.assertListEqual(["abc", ".", "def", ".", ","], res)
    
    def test_splitWestern_fails_on_empty_whitespace(self):
        txt = "A high powered mutant"

        with self.assertRaises(ValueError):
            splitWestern(txt, whitespace="")
    
    def test_splitWestern_whitespace_not_a_string(self):
        txt = "ab1c d1ef"

        lst = list(splitWestern(txt, whitespace=1))
        self.assertListEqual(["ab", "c d", "ef"], lst)<commit_msg>we must evaluate the generator.<commit_after>from wordsalad.text import splitWestern
import unittest

class TestTokenisation(unittest.TestCase):
    
    def test_splitWestern_punctuation_treated_like_one_word(self):
        txt = "abc. def.,"

        res = list(splitWestern(txt))

        self.assertListEqual(["abc", ".", "def", ".", ","], res)
    
    def test_splitWestern_fails_on_empty_whitespace(self):
        txt = "A high powered mutant"

        with self.assertRaises(ValueError):
            list(splitWestern(txt, whitespace=""))
    
    def test_splitWestern_whitespace_not_a_string(self):
        txt = "ab1c d1ef"

        lst = list(splitWestern(txt, whitespace=1))
        self.assertListEqual(["ab", "c d", "ef"], lst)
{'fraction_non_alphanumeric': 0.1226027397260274, 'fraction_numerical': 0.00410958904109589, 'mean_word_length': 3.3224852071005917, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3665616', 'n_tokens_mistral': 489, 'n_tokens_neox': 459, 'n_words': 87}