def test_tokens_ngram_processor_with_template_gaps_basic(self):
    lines = [u'My old {{3 John Doe}} is rich']
    unigrams = unigram_splitter(lines, splitter=template_splitter)
    templated = template_processor(unigrams)
    result = list(tokens_ngram_processor(templated, ngram_len=3))
    expected = [(Token(start=0, start_line=0, start_char=0, end_line=0, end_char=2, end=0, gap=0, value=u'my'), Token(start=0, start_line=0, start_char=3, end_line=0, end_char=6, end=0, gap=3, value=u'old')), (Token(start=0, start_line=0, start_char=22, end_line=0, end_char=24, end=0, gap=0, value=u'is'), Token(start=0, start_line=0, start_char=25, end_line=0, end_char=29, end=0, gap=0, value=u'rich'))]
    assert (expected == result)
def test_tokens_ngram_processor_with_template_gaps_basic(self):
    lines = [u'My old {{3 John Doe}} is rich']
    unigrams = unigram_splitter(lines, splitter=template_splitter)
    templated = template_processor(unigrams)
    result = list(tokens_ngram_processor(templated, ngram_len=3))
    expected = [(Token(start=0, start_line=0, start_char=0, end_line=0, end_char=2, end=0, gap=0, value=u'my'), Token(start=0, start_line=0, start_char=3, end_line=0, end_char=6, end=0, gap=3, value=u'old')), (Token(start=0, start_line=0, start_char=22, end_line=0, end_char=24, end=0, gap=0, value=u'is'), Token(start=0, start_line=0, start_char=25, end_line=0, end_char=29, end=0, gap=0, value=u'rich'))]
    assert (expected == result)