def test_ngram_tokenizer_can_handle_gaps_at_end_of_text(self):
    lines = [u'Neither the name of {{10 the ORGANIZATION}} ']
    ngram_len = 2
    result = list(ngram_tokenizer(lines, ngram_len, template=True))
    expected = [Token(start=0, start_line=0, start_char=0, end_line=0, end_char=11, end=1, gap=0, value=(u'neither', u'the')), Token(start=1, start_line=0, start_char=8, end_line=0, end_char=16, end=2, gap=0, value=(u'the', u'name')), Token(start=2, start_line=0, start_char=12, end_line=0, end_char=19, end=3, gap=10, value=(u'name', u'of'))]
    assert (expected == result)
def test_ngram_tokenizer_can_handle_gaps_at_end_of_text(self):
    lines = [u'Neither the name of {{10 the ORGANIZATION}} ']
    ngram_len = 2
    result = list(ngram_tokenizer(lines, ngram_len, template=True))
    expected = [Token(start=0, start_line=0, start_char=0, end_line=0, end_char=11, end=1, gap=0, value=(u'neither', u'the')), Token(start=1, start_line=0, start_char=8, end_line=0, end_char=16, end=2, gap=0, value=(u'the', u'name')), Token(start=2, start_line=0, start_char=12, end_line=0, end_char=19, end=3, gap=10, value=(u'name', u'of'))]
    assert (expected == result)