def test_tokens_ngram_processor_with_gaps_at_the_end_does_yield_empty_tuples(self):
    lines = [u'My {{3 tailor Joe}} is quite {{ pleasant and }}']
    unigrams = unigram_splitter(lines, splitter=template_splitter)
    templated = template_processor(unigrams)
    ngram_len = 3
    result = list(tokens_ngram_processor(templated, ngram_len=ngram_len))
    assert ((None, None, None) != result[(- 1)])
    expected = [(Token(start=0, start_line=0, start_char=0, end_line=0, end_char=2, end=0, gap=3, value=u'my'),), (Token(start=0, start_line=0, start_char=20, end_line=0, end_char=22, end=0, gap=0, value=u'is'), Token(start=0, start_line=0, start_char=23, end_line=0, end_char=28, end=0, gap=5, value=u'quite'))]
    assert (expected == result)
def test_tokens_ngram_processor_with_gaps_at_the_end_does_yield_empty_tuples(self):
    lines = [u'My {{3 tailor Joe}} is quite {{ pleasant and }}']
    unigrams = unigram_splitter(lines, splitter=template_splitter)
    templated = template_processor(unigrams)
    ngram_len = 3
    result = list(tokens_ngram_processor(templated, ngram_len=ngram_len))
    assert ((None, None, None) != result[(- 1)])
    expected = [(Token(start=0, start_line=0, start_char=0, end_line=0, end_char=2, end=0, gap=3, value=u'my'),), (Token(start=0, start_line=0, start_char=20, end_line=0, end_char=22, end=0, gap=0, value=u'is'), Token(start=0, start_line=0, start_char=23, end_line=0, end_char=28, end=0, gap=5, value=u'quite'))]
    assert (expected == result)