def test_position_processor(self):
    tokens = [Token(value=u'abc'), Token(value=u'def'), Token(value=u'temp'), Token(value=u'ghi')]
    expected = [Token(value=u'abc', start=0, end=0), Token(value=u'def', start=1, end=1), Token(value=u'temp', start=2, end=2), Token(value=u'ghi', start=3, end=3)]
    result = list(position_processor(tokens))
    assert (expected == result)
def test_position_processor(self):
    tokens = [Token(value=u'abc'), Token(value=u'def'), Token(value=u'temp'), Token(value=u'ghi')]
    expected = [Token(value=u'abc', start=0, end=0), Token(value=u'def', start=1, end=1), Token(value=u'temp', start=2, end=2), Token(value=u'ghi', start=3, end=3)]
    result = list(position_processor(tokens))
    assert (expected == result)