def test_unigrams_word_splitter_can_split(self):
    text = iter(u'abc def \n GHI'.splitlines())
    result = list(unigram_splitter(text, splitter=word_splitter))
    expected = [Token(start_line=0, end_line=0, start_char=0, end_char=3, value=u'abc'), Token(start_line=0, end_line=0, start_char=4, end_char=7, value=u'def'), Token(start_line=1, end_line=1, start_char=1, end_char=4, value=u'ghi')]
    assert (expected == result)
def test_unigrams_word_splitter_can_split(self):
    text = iter(u'abc def \n GHI'.splitlines())
    result = list(unigram_splitter(text, splitter=word_splitter))
    expected = [Token(start_line=0, end_line=0, start_char=0, end_char=3, value=u'abc'), Token(start_line=0, end_line=0, start_char=4, end_char=7, value=u'def'), Token(start_line=1, end_line=1, start_char=1, end_char=4, value=u'ghi')]
    assert (expected == result)