def test_tokens_ngram_processor_with_gaps_can_handle_contiguous_template_regions(self):
    lines = u'Neither the name of {{10 the ORGANIZATION}} nor {{}}\n                   {{6 }}of its contributors may materials provided with the\n                   distribution.'.splitlines()
    unigrams = unigram_splitter(lines, splitter=template_splitter)
    templated = template_processor(unigrams)
    result = list(tokens_ngram_processor(templated, ngram_len=4))
    expected = [(Token(start=0, start_line=0, start_char=0, end_line=0, end_char=7, end=0, gap=0, value=u'neither'), Token(start=0, start_line=0, start_char=8, end_line=0, end_char=11, end=0, gap=0, value=u'the'), Token(start=0, start_line=0, start_char=12, end_line=0, end_char=16, end=0, gap=0, value=u'name'), Token(start=0, start_line=0, start_char=17, end_line=0, end_char=19, end=0, gap=10, value=u'of')), (Token(start=0, start_line=0, start_char=44, end_line=0, end_char=47, end=0, gap=5, value=u'nor'),), (Token(start=0, start_line=1, start_char=25, end_line=1, end_char=27, end=0, gap=0, value=u'of'), Token(start=0, start_line=1, start_char=28, end_line=1, end_char=31, end=0, gap=0, value=u'its'), Token(start=0, start_line=1, start_char=32, end_line=1, end_char=44, end=0, gap=0, value=u'contributors'), Token(start=0, start_line=1, start_char=45, end_line=1, end_char=48, end=0, gap=0, value=u'may')), (Token(start=0, start_line=1, start_char=28, end_line=1, end_char=31, end=0, gap=0, value=u'its'), Token(start=0, start_line=1, start_char=32, end_line=1, end_char=44, end=0, gap=0, value=u'contributors'), Token(start=0, start_line=1, start_char=45, end_line=1, end_char=48, end=0, gap=0, value=u'may'), Token(start=0, start_line=1, start_char=49, end_line=1, end_char=58, end=0, gap=0, value=u'materials')), (Token(start=0, start_line=1, start_char=32, end_line=1, end_char=44, end=0, gap=0, value=u'contributors'), Token(start=0, start_line=1, start_char=45, end_line=1, end_char=48, end=0, gap=0, value=u'may'), Token(start=0, start_line=1, start_char=49, end_line=1, end_char=58, end=0, gap=0, value=u'materials'), Token(start=0, start_line=1, start_char=59, end_line=1, end_char=67, end=0, gap=0, value=u'provided')), (Token(start=0, start_line=1, start_char=45, end_line=1, end_char=48, end=0, gap=0, value=u'may'), Token(start=0, start_line=1, start_char=49, end_line=1, end_char=58, end=0, gap=0, value=u'materials'), Token(start=0, start_line=1, start_char=59, end_line=1, end_char=67, end=0, gap=0, value=u'provided'), Token(start=0, start_line=1, start_char=68, end_line=1, end_char=72, end=0, gap=0, value=u'with')), (Token(start=0, start_line=1, start_char=49, end_line=1, end_char=58, end=0, gap=0, value=u'materials'), Token(start=0, start_line=1, start_char=59, end_line=1, end_char=67, end=0, gap=0, value=u'provided'), Token(start=0, start_line=1, start_char=68, end_line=1, end_char=72, end=0, gap=0, value=u'with'), Token(start=0, start_line=1, start_char=73, end_line=1, end_char=76, end=0, gap=0, value=u'the')), (Token(start=0, start_line=1, start_char=59, end_line=1, end_char=67, end=0, gap=0, value=u'provided'), Token(start=0, start_line=1, start_char=68, end_line=1, end_char=72, end=0, gap=0, value=u'with'), Token(start=0, start_line=1, start_char=73, end_line=1, end_char=76, end=0, gap=0, value=u'the'), Token(start=0, start_line=2, start_char=19, end_line=2, end_char=31, end=0, gap=0, value=u'distribution'))]
    assert (expected == result)
def test_tokens_ngram_processor_with_gaps_can_handle_contiguous_template_regions(self):
    lines = u'Neither the name of {{10 the ORGANIZATION}} nor {{}}\n                   {{6 }}of its contributors may materials provided with the\n                   distribution.'.splitlines()
    unigrams = unigram_splitter(lines, splitter=template_splitter)
    templated = template_processor(unigrams)
    result = list(tokens_ngram_processor(templated, ngram_len=4))
    expected = [(Token(start=0, start_line=0, start_char=0, end_line=0, end_char=7, end=0, gap=0, value=u'neither'), Token(start=0, start_line=0, start_char=8, end_line=0, end_char=11, end=0, gap=0, value=u'the'), Token(start=0, start_line=0, start_char=12, end_line=0, end_char=16, end=0, gap=0, value=u'name'), Token(start=0, start_line=0, start_char=17, end_line=0, end_char=19, end=0, gap=10, value=u'of')), (Token(start=0, start_line=0, start_char=44, end_line=0, end_char=47, end=0, gap=5, value=u'nor'),), (Token(start=0, start_line=1, start_char=25, end_line=1, end_char=27, end=0, gap=0, value=u'of'), Token(start=0, start_line=1, start_char=28, end_line=1, end_char=31, end=0, gap=0, value=u'its'), Token(start=0, start_line=1, start_char=32, end_line=1, end_char=44, end=0, gap=0, value=u'contributors'), Token(start=0, start_line=1, start_char=45, end_line=1, end_char=48, end=0, gap=0, value=u'may')), (Token(start=0, start_line=1, start_char=28, end_line=1, end_char=31, end=0, gap=0, value=u'its'), Token(start=0, start_line=1, start_char=32, end_line=1, end_char=44, end=0, gap=0, value=u'contributors'), Token(start=0, start_line=1, start_char=45, end_line=1, end_char=48, end=0, gap=0, value=u'may'), Token(start=0, start_line=1, start_char=49, end_line=1, end_char=58, end=0, gap=0, value=u'materials')), (Token(start=0, start_line=1, start_char=32, end_line=1, end_char=44, end=0, gap=0, value=u'contributors'), Token(start=0, start_line=1, start_char=45, end_line=1, end_char=48, end=0, gap=0, value=u'may'), Token(start=0, start_line=1, start_char=49, end_line=1, end_char=58, end=0, gap=0, value=u'materials'), Token(start=0, start_line=1, start_char=59, end_line=1, end_char=67, end=0, gap=0, value=u'provided')), (Token(start=0, start_line=1, start_char=45, end_line=1, end_char=48, end=0, gap=0, value=u'may'), Token(start=0, start_line=1, start_char=49, end_line=1, end_char=58, end=0, gap=0, value=u'materials'), Token(start=0, start_line=1, start_char=59, end_line=1, end_char=67, end=0, gap=0, value=u'provided'), Token(start=0, start_line=1, start_char=68, end_line=1, end_char=72, end=0, gap=0, value=u'with')), (Token(start=0, start_line=1, start_char=49, end_line=1, end_char=58, end=0, gap=0, value=u'materials'), Token(start=0, start_line=1, start_char=59, end_line=1, end_char=67, end=0, gap=0, value=u'provided'), Token(start=0, start_line=1, start_char=68, end_line=1, end_char=72, end=0, gap=0, value=u'with'), Token(start=0, start_line=1, start_char=73, end_line=1, end_char=76, end=0, gap=0, value=u'the')), (Token(start=0, start_line=1, start_char=59, end_line=1, end_char=67, end=0, gap=0, value=u'provided'), Token(start=0, start_line=1, start_char=68, end_line=1, end_char=72, end=0, gap=0, value=u'with'), Token(start=0, start_line=1, start_char=73, end_line=1, end_char=76, end=0, gap=0, value=u'the'), Token(start=0, start_line=2, start_char=19, end_line=2, end_char=31, end=0, gap=0, value=u'distribution'))]
    assert (expected == result)