def test_tokens_ngram_processor_with_template_gaps_merged(self):
    lines = [u'My old tailor {{3 John Doe}} is quite very rich']
    unigrams = unigram_splitter(lines, splitter=template_splitter)
    templated = template_processor(unigrams)
    ngram_len = 3
    ngrams_tuples = tokens_ngram_processor(templated, ngram_len=ngram_len)
    result = list(ngram_to_token(ngrams_tuples))
    expected = [Token(start_line=0, start_char=0, end_line=0, end_char=13, gap=ngram_len, value=(u'my', u'old', u'tailor')), Token(start_line=0, start_char=29, end_line=0, end_char=42, gap=0, value=(u'is', u'quite', u'very')), Token(start_line=0, start_char=32, end_line=0, end_char=47, gap=0, value=(u'quite', u'very', u'rich'))]
    assert (expected == result)
def test_tokens_ngram_processor_with_template_gaps_merged(self):
    lines = [u'My old tailor {{3 John Doe}} is quite very rich']
    unigrams = unigram_splitter(lines, splitter=template_splitter)
    templated = template_processor(unigrams)
    ngram_len = 3
    ngrams_tuples = tokens_ngram_processor(templated, ngram_len=ngram_len)
    result = list(ngram_to_token(ngrams_tuples))
    expected = [Token(start_line=0, start_char=0, end_line=0, end_char=13, gap=ngram_len, value=(u'my', u'old', u'tailor')), Token(start_line=0, start_char=29, end_line=0, end_char=42, gap=0, value=(u'is', u'quite', u'very')), Token(start_line=0, start_char=32, end_line=0, end_char=47, gap=0, value=(u'quite', u'very', u'rich'))]
    assert (expected == result)