def test_unigrams_template_splitter_can_split_templates(self):
    text = u'abc def \n {{temp}} GHI'.splitlines()
    result = list(unigram_splitter(text, splitter=template_splitter))
    expected = [Token(start_line=0, end_line=0, start_char=0, end_char=3, value=u'abc'), Token(start_line=0, end_line=0, start_char=4, end_char=7, value=u'def'), Token(start_line=1, end_line=1, start_char=1, end_char=3, value=u'{{'), Token(start_line=1, end_line=1, start_char=3, end_char=7, value=u'temp'), Token(start_line=1, end_line=1, start_char=7, end_char=9, value=u'}}'), Token(start_line=1, end_line=1, start_char=10, end_char=13, value=u'ghi')]
    assert (expected == result)
def test_unigrams_template_splitter_can_split_templates(self):
    text = u'abc def \n {{temp}} GHI'.splitlines()
    result = list(unigram_splitter(text, splitter=template_splitter))
    expected = [Token(start_line=0, end_line=0, start_char=0, end_char=3, value=u'abc'), Token(start_line=0, end_line=0, start_char=4, end_char=7, value=u'def'), Token(start_line=1, end_line=1, start_char=1, end_char=3, value=u'{{'), Token(start_line=1, end_line=1, start_char=3, end_char=7, value=u'temp'), Token(start_line=1, end_line=1, start_char=7, end_char=9, value=u'}}'), Token(start_line=1, end_line=1, start_char=10, end_char=13, value=u'ghi')]
    assert (expected == result)