def test_plain_unigrams_from_templated_unigrams(self):
    lines = [u'My old tailor {{3 John Doe}} is quite very rich']
    unigrams = unigram_splitter(lines, splitter=template_splitter)
    result = list(template_processor(unigrams))
    expected = [Token(start=0, start_line=0, start_char=0, end_line=0, end_char=2, end=0, gap=0, value=u'my'), Token(start=0, start_line=0, start_char=3, end_line=0, end_char=6, end=0, gap=0, value=u'old'), Token(start=0, start_line=0, start_char=7, end_line=0, end_char=13, end=0, gap=3, value=u'tailor'), Token(start=0, start_line=0, start_char=29, end_line=0, end_char=31, end=0, gap=0, value=u'is'), Token(start=0, start_line=0, start_char=32, end_line=0, end_char=37, end=0, gap=0, value=u'quite'), Token(start=0, start_line=0, start_char=38, end_line=0, end_char=42, end=0, gap=0, value=u'very'), Token(start=0, start_line=0, start_char=43, end_line=0, end_char=47, end=0, gap=0, value=u'rich')]
    assert (expected == result)
def test_plain_unigrams_from_templated_unigrams(self):
    lines = [u'My old tailor {{3 John Doe}} is quite very rich']
    unigrams = unigram_splitter(lines, splitter=template_splitter)
    result = list(template_processor(unigrams))
    expected = [Token(start=0, start_line=0, start_char=0, end_line=0, end_char=2, end=0, gap=0, value=u'my'), Token(start=0, start_line=0, start_char=3, end_line=0, end_char=6, end=0, gap=0, value=u'old'), Token(start=0, start_line=0, start_char=7, end_line=0, end_char=13, end=0, gap=3, value=u'tailor'), Token(start=0, start_line=0, start_char=29, end_line=0, end_char=31, end=0, gap=0, value=u'is'), Token(start=0, start_line=0, start_char=32, end_line=0, end_char=37, end=0, gap=0, value=u'quite'), Token(start=0, start_line=0, start_char=38, end_line=0, end_char=42, end=0, gap=0, value=u'very'), Token(start=0, start_line=0, start_char=43, end_line=0, end_char=47, end=0, gap=0, value=u'rich')]
    assert (expected == result)