def test_ngram_tokenizer_returns_correct_offsets_template(self):
    lines = [u'X11 License']
    ngram_len = 3
    result = list(ngram_tokenizer(lines, ngram_len, template=True))
    assert (lines == list(doc_subset(lines, result[0])))
    expected = [Token(start=0, start_line=0, start_char=0, end_line=0, end_char=11, end=1, gap=0, value=(u'x11', u'license'))]
    assert (expected == result)
def test_ngram_tokenizer_returns_correct_offsets_template(self):
    lines = [u'X11 License']
    ngram_len = 3
    result = list(ngram_tokenizer(lines, ngram_len, template=True))
    assert (lines == list(doc_subset(lines, result[0])))
    expected = [Token(start=0, start_line=0, start_char=0, end_line=0, end_char=11, end=1, gap=0, value=(u'x11', u'license'))]
    assert (expected == result)