def test_tokens_ngram_processor_with_gaps_merged_always_returns_4grams_when_requested(self):
    lines = u'Neither the name of {{10 the ORGANIZATION}} nor {{}}the\n                   names {{}}of its contributors may materials provided with\n                   the distribution.'.splitlines()
    unigrams = unigram_splitter(lines, splitter=template_splitter)
    templated = template_processor(unigrams)
    result = list(tokens_ngram_processor(templated, ngram_len=4))
    expected = [(Token(start=0, start_line=0, start_char=0, end_line=0, end_char=7, end=0, gap=0, value=u'neither'), Token(start=0, start_line=0, start_char=8, end_line=0, end_char=11, end=0, gap=0, value=u'the'), Token(start=0, start_line=0, start_char=12, end_line=0, end_char=16, end=0, gap=0, value=u'name'), Token(start=0, start_line=0, start_char=17, end_line=0, end_char=19, end=0, gap=10, value=u'of')), (Token(start=0, start_line=0, start_char=44, end_line=0, end_char=47, end=0, gap=5, value=u'nor'),), (Token(start=0, start_line=0, start_char=52, end_line=0, end_char=55, end=0, gap=0, value=u'the'), Token(start=0, start_line=1, start_char=19, end_line=1, end_char=24, end=0, gap=5, value=u'names')), (Token(start=0, start_line=1, start_char=29, end_line=1, end_char=31, end=0, gap=0, value=u'of'), Token(start=0, start_line=1, start_char=32, end_line=1, end_char=35, end=0, gap=0, value=u'its'), Token(start=0, start_line=1, start_char=36, end_line=1, end_char=48, end=0, gap=0, value=u'contributors'), Token(start=0, start_line=1, start_char=49, end_line=1, end_char=52, end=0, gap=0, value=u'may')), (Token(start=0, start_line=1, start_char=32, end_line=1, end_char=35, end=0, gap=0, value=u'its'), Token(start=0, start_line=1, start_char=36, end_line=1, end_char=48, end=0, gap=0, value=u'contributors'), Token(start=0, start_line=1, start_char=49, end_line=1, end_char=52, end=0, gap=0, value=u'may'), Token(start=0, start_line=1, start_char=53, end_line=1, end_char=62, end=0, gap=0, value=u'materials')), (Token(start=0, start_line=1, start_char=36, end_line=1, end_char=48, end=0, gap=0, value=u'contributors'), Token(start=0, start_line=1, start_char=49, end_line=1, end_char=52, end=0, gap=0, value=u'may'), Token(start=0, start_line=1, start_char=53, end_line=1, end_char=62, end=0, gap=0, value=u'materials'), Token(start=0, start_line=1, start_char=63, end_line=1, end_char=71, end=0, gap=0, value=u'provided')), (Token(start=0, start_line=1, start_char=49, end_line=1, end_char=52, end=0, gap=0, value=u'may'), Token(start=0, start_line=1, start_char=53, end_line=1, end_char=62, end=0, gap=0, value=u'materials'), Token(start=0, start_line=1, start_char=63, end_line=1, end_char=71, end=0, gap=0, value=u'provided'), Token(start=0, start_line=1, start_char=72, end_line=1, end_char=76, end=0, gap=0, value=u'with')), (Token(start=0, start_line=1, start_char=53, end_line=1, end_char=62, end=0, gap=0, value=u'materials'), Token(start=0, start_line=1, start_char=63, end_line=1, end_char=71, end=0, gap=0, value=u'provided'), Token(start=0, start_line=1, start_char=72, end_line=1, end_char=76, end=0, gap=0, value=u'with'), Token(start=0, start_line=2, start_char=19, end_line=2, end_char=22, end=0, gap=0, value=u'the')), (Token(start=0, start_line=1, start_char=63, end_line=1, end_char=71, end=0, gap=0, value=u'provided'), Token(start=0, start_line=1, start_char=72, end_line=1, end_char=76, end=0, gap=0, value=u'with'), Token(start=0, start_line=2, start_char=19, end_line=2, end_char=22, end=0, gap=0, value=u'the'), Token(start=0, start_line=2, start_char=23, end_line=2, end_char=35, end=0, gap=0, value=u'distribution'))]
    assert (expected == result)
def test_tokens_ngram_processor_with_gaps_merged_always_returns_4grams_when_requested(self):
    lines = u'Neither the name of {{10 the ORGANIZATION}} nor {{}}the\n                   names {{}}of its contributors may materials provided with\n                   the distribution.'.splitlines()
    unigrams = unigram_splitter(lines, splitter=template_splitter)
    templated = template_processor(unigrams)
    result = list(tokens_ngram_processor(templated, ngram_len=4))
    expected = [(Token(start=0, start_line=0, start_char=0, end_line=0, end_char=7, end=0, gap=0, value=u'neither'), Token(start=0, start_line=0, start_char=8, end_line=0, end_char=11, end=0, gap=0, value=u'the'), Token(start=0, start_line=0, start_char=12, end_line=0, end_char=16, end=0, gap=0, value=u'name'), Token(start=0, start_line=0, start_char=17, end_line=0, end_char=19, end=0, gap=10, value=u'of')), (Token(start=0, start_line=0, start_char=44, end_line=0, end_char=47, end=0, gap=5, value=u'nor'),), (Token(start=0, start_line=0, start_char=52, end_line=0, end_char=55, end=0, gap=0, value=u'the'), Token(start=0, start_line=1, start_char=19, end_line=1, end_char=24, end=0, gap=5, value=u'names')), (Token(start=0, start_line=1, start_char=29, end_line=1, end_char=31, end=0, gap=0, value=u'of'), Token(start=0, start_line=1, start_char=32, end_line=1, end_char=35, end=0, gap=0, value=u'its'), Token(start=0, start_line=1, start_char=36, end_line=1, end_char=48, end=0, gap=0, value=u'contributors'), Token(start=0, start_line=1, start_char=49, end_line=1, end_char=52, end=0, gap=0, value=u'may')), (Token(start=0, start_line=1, start_char=32, end_line=1, end_char=35, end=0, gap=0, value=u'its'), Token(start=0, start_line=1, start_char=36, end_line=1, end_char=48, end=0, gap=0, value=u'contributors'), Token(start=0, start_line=1, start_char=49, end_line=1, end_char=52, end=0, gap=0, value=u'may'), Token(start=0, start_line=1, start_char=53, end_line=1, end_char=62, end=0, gap=0, value=u'materials')), (Token(start=0, start_line=1, start_char=36, end_line=1, end_char=48, end=0, gap=0, value=u'contributors'), Token(start=0, start_line=1, start_char=49, end_line=1, end_char=52, end=0, gap=0, value=u'may'), Token(start=0, start_line=1, start_char=53, end_line=1, end_char=62, end=0, gap=0, value=u'materials'), Token(start=0, start_line=1, start_char=63, end_line=1, end_char=71, end=0, gap=0, value=u'provided')), (Token(start=0, start_line=1, start_char=49, end_line=1, end_char=52, end=0, gap=0, value=u'may'), Token(start=0, start_line=1, start_char=53, end_line=1, end_char=62, end=0, gap=0, value=u'materials'), Token(start=0, start_line=1, start_char=63, end_line=1, end_char=71, end=0, gap=0, value=u'provided'), Token(start=0, start_line=1, start_char=72, end_line=1, end_char=76, end=0, gap=0, value=u'with')), (Token(start=0, start_line=1, start_char=53, end_line=1, end_char=62, end=0, gap=0, value=u'materials'), Token(start=0, start_line=1, start_char=63, end_line=1, end_char=71, end=0, gap=0, value=u'provided'), Token(start=0, start_line=1, start_char=72, end_line=1, end_char=76, end=0, gap=0, value=u'with'), Token(start=0, start_line=2, start_char=19, end_line=2, end_char=22, end=0, gap=0, value=u'the')), (Token(start=0, start_line=1, start_char=63, end_line=1, end_char=71, end=0, gap=0, value=u'provided'), Token(start=0, start_line=1, start_char=72, end_line=1, end_char=76, end=0, gap=0, value=u'with'), Token(start=0, start_line=2, start_char=19, end_line=2, end_char=22, end=0, gap=0, value=u'the'), Token(start=0, start_line=2, start_char=23, end_line=2, end_char=35, end=0, gap=0, value=u'distribution'))]
    assert (expected == result)