def test_tokens_ngram_processor_n10_with_2_tokens(self):
    text = u'this is'
    unigrams = list(unigram_splitter(text.splitlines()))
    expected = [(Token(start_line=0, start_char=0, end_line=0, end_char=4, value=u'this'), Token(start_line=0, start_char=5, end_line=0, end_char=7, value=u'is'))]
    result = list(tokens_ngram_processor(iter(unigrams), ngram_len=10))
    assert (expected == result)
def test_tokens_ngram_processor_n10_with_2_tokens(self):
    text = u'this is'
    unigrams = list(unigram_splitter(text.splitlines()))
    expected = [(Token(start_line=0, start_char=0, end_line=0, end_char=4, value=u'this'), Token(start_line=0, start_char=5, end_line=0, end_char=7, value=u'is'))]
    result = list(tokens_ngram_processor(iter(unigrams), ngram_len=10))
    assert (expected == result)