def test_tokens_ngram_processor_3grams_from_unigrams_on_multilines(self):
    text = u'this is some text \n on multiple lines'
    unigrams = unigram_splitter(text.splitlines())
    result = list(tokens_ngram_processor(unigrams, ngram_len=3))
    expected = [(Token(start_line=0, start_char=0, end_line=0, end_char=4, value=u'this'), Token(start_line=0, start_char=5, end_line=0, end_char=7, value=u'is'), Token(start_line=0, start_char=8, end_line=0, end_char=12, value=u'some')), (Token(start_line=0, start_char=5, end_line=0, end_char=7, value=u'is'), Token(start_line=0, start_char=8, end_line=0, end_char=12, value=u'some'), Token(start_line=0, start_char=13, end_line=0, end_char=17, value=u'text')), (Token(start_line=0, start_char=8, end_line=0, end_char=12, value=u'some'), Token(start_line=0, start_char=13, end_line=0, end_char=17, value=u'text'), Token(start_line=1, start_char=1, end_line=1, end_char=3, value=u'on')), (Token(start_line=0, start_char=13, end_line=0, end_char=17, value=u'text'), Token(start_line=1, start_char=1, end_line=1, end_char=3, value=u'on'), Token(start_line=1, start_char=4, end_line=1, end_char=12, value=u'multiple')), (Token(start_line=1, start_char=1, end_line=1, end_char=3, value=u'on'), Token(start_line=1, start_char=4, end_line=1, end_char=12, value=u'multiple'), Token(start_line=1, start_char=13, end_line=1, end_char=18, value=u'lines'))]
    assert (expected == result)
def test_tokens_ngram_processor_3grams_from_unigrams_on_multilines(self):
    text = u'this is some text \n on multiple lines'
    unigrams = unigram_splitter(text.splitlines())
    result = list(tokens_ngram_processor(unigrams, ngram_len=3))
    expected = [(Token(start_line=0, start_char=0, end_line=0, end_char=4, value=u'this'), Token(start_line=0, start_char=5, end_line=0, end_char=7, value=u'is'), Token(start_line=0, start_char=8, end_line=0, end_char=12, value=u'some')), (Token(start_line=0, start_char=5, end_line=0, end_char=7, value=u'is'), Token(start_line=0, start_char=8, end_line=0, end_char=12, value=u'some'), Token(start_line=0, start_char=13, end_line=0, end_char=17, value=u'text')), (Token(start_line=0, start_char=8, end_line=0, end_char=12, value=u'some'), Token(start_line=0, start_char=13, end_line=0, end_char=17, value=u'text'), Token(start_line=1, start_char=1, end_line=1, end_char=3, value=u'on')), (Token(start_line=0, start_char=13, end_line=0, end_char=17, value=u'text'), Token(start_line=1, start_char=1, end_line=1, end_char=3, value=u'on'), Token(start_line=1, start_char=4, end_line=1, end_char=12, value=u'multiple')), (Token(start_line=1, start_char=1, end_line=1, end_char=3, value=u'on'), Token(start_line=1, start_char=4, end_line=1, end_char=12, value=u'multiple'), Token(start_line=1, start_char=13, end_line=1, end_char=18, value=u'lines'))]
    assert (expected == result)