def test_tokens_ngram_processor_with_gaps_merged_short_grams(self):
    lines = [u'My {{3 tailor Joe}} is quite {{ pleasant and }} very rich']
    unigrams = unigram_splitter(lines, splitter=template_splitter)
    templated = template_processor(unigrams)
    ngram_len = 3
    ngrams_tuples = tokens_ngram_processor(templated, ngram_len=ngram_len)
    result = list(ngram_to_token(ngrams_tuples))
    expected = [Token(start=0, start_line=0, start_char=0, end_line=0, end_char=2, end=0, gap=3, value=(u'my',)), Token(start=0, start_line=0, start_char=20, end_line=0, end_char=28, end=0, gap=5, value=(u'is', u'quite')), Token(start=0, start_line=0, start_char=48, end_line=0, end_char=57, end=0, gap=0, value=(u'very', u'rich'))]
    assert (expected == result)
def test_tokens_ngram_processor_with_gaps_merged_short_grams(self):
    lines = [u'My {{3 tailor Joe}} is quite {{ pleasant and }} very rich']
    unigrams = unigram_splitter(lines, splitter=template_splitter)
    templated = template_processor(unigrams)
    ngram_len = 3
    ngrams_tuples = tokens_ngram_processor(templated, ngram_len=ngram_len)
    result = list(ngram_to_token(ngrams_tuples))
    expected = [Token(start=0, start_line=0, start_char=0, end_line=0, end_char=2, end=0, gap=3, value=(u'my',)), Token(start=0, start_line=0, start_char=20, end_line=0, end_char=28, end=0, gap=5, value=(u'is', u'quite')), Token(start=0, start_line=0, start_char=48, end_line=0, end_char=57, end=0, gap=0, value=(u'very', u'rich'))]
    assert (expected == result)