def test_token_positions_are_kept_same_for_unigrams_and_ngrams_with_template(self):
    lines = u'some text is some text {{ }} in all cases\n \n'
    unigrams = unigram_tokenizer(iter([lines]), template=False)
    tunigrams = unigram_tokenizer(iter([lines]), template=True)
    ngrams = ngram_tokenizer(iter([lines]), ngram_len=3, template=False)
    tngrams = ngram_tokenizer(iter([lines]), ngram_len=3, template=True)
    expected_start_end = (0, 7)

    def check_start_end(l):
        l = list(l)
        result = (l[0].start, l[(- 1)].end)
        assert (expected_start_end == result)
    check_start_end(unigrams)
    check_start_end(tunigrams)
    check_start_end(ngrams)
    check_start_end(tngrams)
def test_token_positions_are_kept_same_for_unigrams_and_ngrams_with_template(self):
    lines = u'some text is some text {{ }} in all cases\n \n'
    unigrams = unigram_tokenizer(iter([lines]), template=False)
    tunigrams = unigram_tokenizer(iter([lines]), template=True)
    ngrams = ngram_tokenizer(iter([lines]), ngram_len=3, template=False)
    tngrams = ngram_tokenizer(iter([lines]), ngram_len=3, template=True)
    expected_start_end = (0, 7)

    def check_start_end(l):
        l = list(l)
        result = (l[0].start, l[(- 1)].end)
        assert (expected_start_end == result)
    check_start_end(unigrams)
    check_start_end(tunigrams)
    check_start_end(ngrams)
    check_start_end(tngrams)