def test_ngram_tokenizer_returns_correct_offsets_n1(self):
    lines = [u'X11 License']
    ngram_len = 1
    result = list(ngram_tokenizer(lines, ngram_len))
    expected = [Token(start=0, start_line=0, start_char=0, end_line=0, end_char=3, end=0, gap=0, value=(u'x11',)), Token(start=1, start_line=0, start_char=4, end_line=0, end_char=11, end=1, gap=0, value=(u'license',))]
    assert (expected == result)
def test_ngram_tokenizer_returns_correct_offsets_n1(self):
    lines = [u'X11 License']
    ngram_len = 1
    result = list(ngram_tokenizer(lines, ngram_len))
    expected = [Token(start=0, start_line=0, start_char=0, end_line=0, end_char=3, end=0, gap=0, value=(u'x11',)), Token(start=1, start_line=0, start_char=4, end_line=0, end_char=11, end=1, gap=0, value=(u'license',))]
    assert (expected == result)