def test_process_template_can_handle_long_text(self):
    expected = [Token(start_line=0, end_line=0, start_char=14, end_char=17, value=u'ist', gap=NO_GAP), Token(start_line=0, end_line=0, start_char=18, end_char=23, value=u'freie', gap=NO_GAP), Token(start_line=0, end_line=0, start_char=24, end_char=32, value=u'software', gap=NO_GAP), Token(start_line=0, end_line=0, start_char=34, end_char=37, value=u'sie', gap=NO_GAP), Token(start_line=0, end_line=0, start_char=38, end_char=44, value=u'können', gap=NO_GAP), Token(start_line=0, end_line=0, start_char=45, end_char=47, value=u'es', gap=NO_GAP), Token(start_line=0, end_line=0, start_char=48, end_char=53, value=u'unter', gap=NO_GAP), Token(start_line=0, end_line=0, start_char=54, end_char=57, value=u'den', gap=NO_GAP), Token(start_line=0, end_line=0, start_char=58, end_char=69, value=u'bedingungen', gap=NO_GAP), Token(start_line=0, end_line=0, start_char=70, end_char=73, value=u'der', gap=NO_GAP), Token(start_line=0, end_line=0, start_char=74, end_char=77, value=u'gnu', gap=NO_GAP), Token(start_line=1, end_line=1, start_char=1, end_char=8, value=u'general', gap=NO_GAP), Token(start_line=1, end_line=1, start_char=10, end_char=11, value=u'n', gap=NO_GAP), Token(start_line=2, end_line=2, start_char=1, end_char=7, value=u'public', gap=NO_GAP), Token(start_line=2, end_line=2, start_char=8, end_char=15, value=u'license', gap=NO_GAP), Token(start_line=2, end_line=2, start_char=17, end_char=20, value=u'wie', gap=NO_GAP), Token(start_line=2, end_line=2, start_char=21, end_char=24, value=u'von', gap=NO_GAP), Token(start_line=2, end_line=2, start_char=25, end_char=28, value=u'der', gap=NO_GAP), Token(start_line=2, end_line=2, start_char=29, end_char=33, value=u'free', gap=NO_GAP), Token(start_line=2, end_line=2, start_char=34, end_char=42, value=u'software', gap=NO_GAP), Token(start_line=2, end_line=2, start_char=43, end_char=53, value=u'foundation', gap=NO_GAP), Token(start_line=2, end_line=2, start_char=54, end_char=68, value=u'veröffentlicht', gap=NO_GAP), Token(start_line=3, end_line=3, start_char=1, end_char=12, value=u'weitergeben', gap=NO_GAP), Token(start_line=3, end_line=3, start_char=13, end_char=16, value=u'und', gap=NO_GAP), Token(start_line=3, end_line=3, start_char=17, end_char=21, value=u'oder', gap=NO_GAP), Token(start_line=3, end_line=3, start_char=23, end_char=24, value=u'n', gap=NO_GAP), Token(start_line=4, end_line=4, start_char=1, end_char=13, value=u'modifizieren', gap=NO_GAP), Token(start_line=4, end_line=4, start_char=15, end_char=23, value=u'entweder', gap=NO_GAP), Token(start_line=4, end_line=4, start_char=24, end_char=29, value=u'gemäß', gap=NO_GAP), Token(start_line=4, end_line=4, start_char=30, end_char=37, value=u'version', gap=NO_GAP), Token(start_line=4, end_line=4, start_char=38, end_char=39, value=u'3', gap=NO_GAP), Token(start_line=4, end_line=4, start_char=40, end_char=43, value=u'der', gap=NO_GAP), Token(start_line=4, end_line=4, start_char=44, end_char=50, value=u'lizenz', gap=NO_GAP), Token(start_line=4, end_line=4, start_char=51, end_char=55, value=u'oder', gap=NO_GAP), Token(start_line=4, end_line=4, start_char=57, end_char=61, value=u'nach', gap=NO_GAP), Token(start_line=4, end_line=4, start_char=62, end_char=67, value=u'ihrer', gap=NO_GAP), Token(start_line=4, end_line=4, start_char=68, end_char=74, value=u'option', gap=NO_GAP), Token(start_line=5, end_line=5, start_char=1, end_char=6, value=u'jeder', gap=NO_GAP), Token(start_line=5, end_line=5, start_char=7, end_char=15, value=u'späteren', gap=NO_GAP), Token(start_line=5, end_line=5, start_char=17, end_char=18, value=u'n', gap=NO_GAP), Token(start_line=6, end_line=6, start_char=1, end_char=8, value=u'version', gap=NO_GAP), Token(start_line=6, end_line=6, start_char=10, end_char=11, value=u'n', gap=NO_GAP), Token(start_line=7, end_line=7, start_char=2, end_char=3, value=u'n', gap=NO_GAP), Token(start_line=8, end_line=8, start_char=1, end_char=4, value=u'die', gap=NO_GAP), Token(start_line=8, end_line=8, start_char=5, end_char=21, value=u'veröffentlichung', gap=NO_GAP), Token(start_line=8, end_line=8, start_char=22, end_char=25, value=u'von', gap=DEFAULT_GAP), Token(start_line=8, end_line=8, start_char=38, end_char=45, value=u'erfolgt', gap=NO_GAP), Token(start_line=8, end_line=8, start_char=46, end_char=48, value=u'in', gap=NO_GAP), Token(start_line=8, end_line=8, start_char=49, end_char=52, value=u'der', gap=NO_GAP), Token(start_line=8, end_line=8, start_char=53, end_char=61, value=u'hoffnung', gap=NO_GAP), Token(start_line=8, end_line=8, start_char=63, end_char=66, value=u'daß', gap=NO_GAP), Token(start_line=8, end_line=8, start_char=67, end_char=69, value=u'es', gap=NO_GAP), Token(start_line=8, end_line=8, start_char=70, end_char=75, value=u'ihnen', gap=NO_GAP), Token(start_line=8, end_line=8, start_char=76, end_char=79, value=u'von', gap=NO_GAP), Token(start_line=9, end_line=9, start_char=1, end_char=7, value=u'nutzen', gap=NO_GAP), Token(start_line=9, end_line=9, start_char=9, end_char=10, value=u'n', gap=NO_GAP), Token(start_line=10, end_line=10, start_char=1, end_char=5, value=u'sein', gap=NO_GAP), Token(start_line=10, end_line=10, start_char=6, end_char=10, value=u'wird', gap=NO_GAP), Token(start_line=10, end_line=10, start_char=12, end_char=16, value=u'aber', gap=NO_GAP), Token(start_line=10, end_line=10, start_char=17, end_char=21, value=u'ohne', gap=NO_GAP), Token(start_line=10, end_line=10, start_char=22, end_char=32, value=u'irgendeine', gap=NO_GAP), Token(start_line=10, end_line=10, start_char=33, end_char=41, value=u'garantie', gap=NO_GAP), Token(start_line=10, end_line=10, start_char=43, end_char=48, value=u'sogar', gap=NO_GAP), Token(start_line=10, end_line=10, start_char=49, end_char=53, value=u'ohne', gap=NO_GAP), Token(start_line=10, end_line=10, start_char=54, end_char=57, value=u'die', gap=NO_GAP), Token(start_line=10, end_line=10, start_char=58, end_char=67, value=u'implizite', gap=NO_GAP), Token(start_line=10, end_line=10, start_char=68, end_char=76, value=u'garantie', gap=NO_GAP), Token(start_line=11, end_line=11, start_char=1, end_char=4, value=u'der', gap=NO_GAP), Token(start_line=11, end_line=11, start_char=5, end_char=15, value=u'marktreife', gap=NO_GAP), Token(start_line=11, end_line=11, start_char=17, end_char=18, value=u'n', gap=NO_GAP), Token(start_line=12, end_line=12, start_char=1, end_char=5, value=u'oder', gap=NO_GAP), Token(start_line=12, end_line=12, start_char=6, end_char=9, value=u'der', gap=NO_GAP), Token(start_line=12, end_line=12, start_char=10, end_char=24, value=u'verwendbarkeit', gap=NO_GAP), Token(start_line=12, end_line=12, start_char=25, end_char=28, value=u'für', gap=NO_GAP), Token(start_line=12, end_line=12, start_char=29, end_char=34, value=u'einen', gap=NO_GAP), Token(start_line=12, end_line=12, start_char=35, end_char=45, value=u'bestimmten', gap=NO_GAP), Token(start_line=12, end_line=12, start_char=46, end_char=51, value=u'zweck', gap=NO_GAP), Token(start_line=12, end_line=12, start_char=53, end_char=60, value=u'details', gap=NO_GAP), Token(start_line=12, end_line=12, start_char=61, end_char=67, value=u'finden', gap=NO_GAP), Token(start_line=12, end_line=12, start_char=68, end_char=71, value=u'sie', gap=NO_GAP), Token(start_line=12, end_line=12, start_char=72, end_char=74, value=u'in', gap=NO_GAP), Token(start_line=13, end_line=13, start_char=1, end_char=4, value=u'der', gap=NO_GAP), Token(start_line=13, end_line=13, start_char=5, end_char=8, value=u'gnu', gap=NO_GAP), Token(start_line=13, end_line=13, start_char=9, end_char=16, value=u'general', gap=NO_GAP), Token(start_line=13, end_line=13, start_char=18, end_char=19, value=u'n', gap=NO_GAP), Token(start_line=14, end_line=14, start_char=1, end_char=7, value=u'public', gap=NO_GAP), Token(start_line=14, end_line=14, start_char=8, end_char=15, value=u'license', gap=NO_GAP), Token(start_line=14, end_line=14, start_char=17, end_char=18, value=u'n', gap=NO_GAP), Token(start_line=15, end_line=15, start_char=2, end_char=3, value=u'n', gap=NO_GAP), Token(start_line=16, end_line=16, start_char=1, end_char=4, value=u'sie', gap=NO_GAP), Token(start_line=16, end_line=16, start_char=5, end_char=12, value=u'sollten', gap=NO_GAP), Token(start_line=16, end_line=16, start_char=13, end_char=16, value=u'ein', gap=NO_GAP), Token(start_line=16, end_line=16, start_char=17, end_char=25, value=u'exemplar', gap=NO_GAP), Token(start_line=16, end_line=16, start_char=26, end_char=29, value=u'der', gap=NO_GAP), Token(start_line=16, end_line=16, start_char=30, end_char=33, value=u'gnu', gap=NO_GAP), Token(start_line=16, end_line=16, start_char=34, end_char=41, value=u'general', gap=NO_GAP), Token(start_line=16, end_line=16, start_char=42, end_char=48, value=u'public', gap=NO_GAP), Token(start_line=16, end_line=16, start_char=49, end_char=56, value=u'license', gap=NO_GAP), Token(start_line=16, end_line=16, start_char=57, end_char=65, value=u'zusammen', gap=NO_GAP), Token(start_line=16, end_line=16, start_char=66, end_char=69, value=u'mit', gap=DEFAULT_GAP), Token(start_line=17, end_line=17, start_char=2, end_char=3, value=u'n', gap=NO_GAP), Token(start_line=18, end_line=18, start_char=1, end_char=9, value=u'erhalten', gap=NO_GAP), Token(start_line=18, end_line=18, start_char=10, end_char=15, value=u'haben', gap=NO_GAP), Token(start_line=18, end_line=18, start_char=17, end_char=22, value=u'falls', gap=NO_GAP), Token(start_line=18, end_line=18, start_char=23, end_char=28, value=u'nicht', gap=NO_GAP), Token(start_line=18, end_line=18, start_char=30, end_char=39, value=u'schreiben', gap=NO_GAP), Token(start_line=18, end_line=18, start_char=40, end_char=43, value=u'sie', gap=NO_GAP), Token(start_line=18, end_line=18, start_char=44, end_char=46, value=u'an', gap=NO_GAP), Token(start_line=18, end_line=18, start_char=47, end_char=50, value=u'die', gap=NO_GAP), Token(start_line=18, end_line=18, start_char=51, end_char=55, value=u'free', gap=NO_GAP), Token(start_line=18, end_line=18, start_char=56, end_char=64, value=u'software', gap=NO_GAP), Token(start_line=18, end_line=18, start_char=65, end_char=75, value=u'foundation', gap=NO_GAP), Token(start_line=19, end_line=19, start_char=2, end_char=3, value=u'n', gap=NO_GAP), Token(start_line=20, end_line=20, start_char=1, end_char=4, value=u'inc', gap=NO_GAP), Token(start_line=20, end_line=20, start_char=7, end_char=9, value=u'51', gap=NO_GAP), Token(start_line=20, end_line=20, start_char=10, end_char=18, value=u'franklin', gap=NO_GAP), Token(start_line=20, end_line=20, start_char=19, end_char=21, value=u'st', gap=NO_GAP), Token(start_line=20, end_line=20, start_char=23, end_char=28, value=u'fifth', gap=NO_GAP), Token(start_line=20, end_line=20, start_char=29, end_char=34, value=u'floor', gap=NO_GAP), Token(start_line=20, end_line=20, start_char=36, end_char=42, value=u'boston', gap=NO_GAP), Token(start_line=20, end_line=20, start_char=44, end_char=46, value=u'ma', gap=NO_GAP), Token(start_line=20, end_line=20, start_char=47, end_char=52, value=u'02110', gap=NO_GAP), Token(start_line=20, end_line=20, start_char=54, end_char=57, value=u'usa', gap=NO_GAP)]
    test_file = self.get_test_loc('analysis/unicode/12180.txt')
    with codecs.open(test_file, encoding='utf-8') as test:
        result = list(self.template_parsing(test))
        assert (expected == result)
def test_process_template_can_handle_long_text(self):
    expected = [Token(start_line=0, end_line=0, start_char=14, end_char=17, value=u'ist', gap=NO_GAP), Token(start_line=0, end_line=0, start_char=18, end_char=23, value=u'freie', gap=NO_GAP), Token(start_line=0, end_line=0, start_char=24, end_char=32, value=u'software', gap=NO_GAP), Token(start_line=0, end_line=0, start_char=34, end_char=37, value=u'sie', gap=NO_GAP), Token(start_line=0, end_line=0, start_char=38, end_char=44, value=u'können', gap=NO_GAP), Token(start_line=0, end_line=0, start_char=45, end_char=47, value=u'es', gap=NO_GAP), Token(start_line=0, end_line=0, start_char=48, end_char=53, value=u'unter', gap=NO_GAP), Token(start_line=0, end_line=0, start_char=54, end_char=57, value=u'den', gap=NO_GAP), Token(start_line=0, end_line=0, start_char=58, end_char=69, value=u'bedingungen', gap=NO_GAP), Token(start_line=0, end_line=0, start_char=70, end_char=73, value=u'der', gap=NO_GAP), Token(start_line=0, end_line=0, start_char=74, end_char=77, value=u'gnu', gap=NO_GAP), Token(start_line=1, end_line=1, start_char=1, end_char=8, value=u'general', gap=NO_GAP), Token(start_line=1, end_line=1, start_char=10, end_char=11, value=u'n', gap=NO_GAP), Token(start_line=2, end_line=2, start_char=1, end_char=7, value=u'public', gap=NO_GAP), Token(start_line=2, end_line=2, start_char=8, end_char=15, value=u'license', gap=NO_GAP), Token(start_line=2, end_line=2, start_char=17, end_char=20, value=u'wie', gap=NO_GAP), Token(start_line=2, end_line=2, start_char=21, end_char=24, value=u'von', gap=NO_GAP), Token(start_line=2, end_line=2, start_char=25, end_char=28, value=u'der', gap=NO_GAP), Token(start_line=2, end_line=2, start_char=29, end_char=33, value=u'free', gap=NO_GAP), Token(start_line=2, end_line=2, start_char=34, end_char=42, value=u'software', gap=NO_GAP), Token(start_line=2, end_line=2, start_char=43, end_char=53, value=u'foundation', gap=NO_GAP), Token(start_line=2, end_line=2, start_char=54, end_char=68, value=u'veröffentlicht', gap=NO_GAP), Token(start_line=3, end_line=3, start_char=1, end_char=12, value=u'weitergeben', gap=NO_GAP), Token(start_line=3, end_line=3, start_char=13, end_char=16, value=u'und', gap=NO_GAP), Token(start_line=3, end_line=3, start_char=17, end_char=21, value=u'oder', gap=NO_GAP), Token(start_line=3, end_line=3, start_char=23, end_char=24, value=u'n', gap=NO_GAP), Token(start_line=4, end_line=4, start_char=1, end_char=13, value=u'modifizieren', gap=NO_GAP), Token(start_line=4, end_line=4, start_char=15, end_char=23, value=u'entweder', gap=NO_GAP), Token(start_line=4, end_line=4, start_char=24, end_char=29, value=u'gemäß', gap=NO_GAP), Token(start_line=4, end_line=4, start_char=30, end_char=37, value=u'version', gap=NO_GAP), Token(start_line=4, end_line=4, start_char=38, end_char=39, value=u'3', gap=NO_GAP), Token(start_line=4, end_line=4, start_char=40, end_char=43, value=u'der', gap=NO_GAP), Token(start_line=4, end_line=4, start_char=44, end_char=50, value=u'lizenz', gap=NO_GAP), Token(start_line=4, end_line=4, start_char=51, end_char=55, value=u'oder', gap=NO_GAP), Token(start_line=4, end_line=4, start_char=57, end_char=61, value=u'nach', gap=NO_GAP), Token(start_line=4, end_line=4, start_char=62, end_char=67, value=u'ihrer', gap=NO_GAP), Token(start_line=4, end_line=4, start_char=68, end_char=74, value=u'option', gap=NO_GAP), Token(start_line=5, end_line=5, start_char=1, end_char=6, value=u'jeder', gap=NO_GAP), Token(start_line=5, end_line=5, start_char=7, end_char=15, value=u'späteren', gap=NO_GAP), Token(start_line=5, end_line=5, start_char=17, end_char=18, value=u'n', gap=NO_GAP), Token(start_line=6, end_line=6, start_char=1, end_char=8, value=u'version', gap=NO_GAP), Token(start_line=6, end_line=6, start_char=10, end_char=11, value=u'n', gap=NO_GAP), Token(start_line=7, end_line=7, start_char=2, end_char=3, value=u'n', gap=NO_GAP), Token(start_line=8, end_line=8, start_char=1, end_char=4, value=u'die', gap=NO_GAP), Token(start_line=8, end_line=8, start_char=5, end_char=21, value=u'veröffentlichung', gap=NO_GAP), Token(start_line=8, end_line=8, start_char=22, end_char=25, value=u'von', gap=DEFAULT_GAP), Token(start_line=8, end_line=8, start_char=38, end_char=45, value=u'erfolgt', gap=NO_GAP), Token(start_line=8, end_line=8, start_char=46, end_char=48, value=u'in', gap=NO_GAP), Token(start_line=8, end_line=8, start_char=49, end_char=52, value=u'der', gap=NO_GAP), Token(start_line=8, end_line=8, start_char=53, end_char=61, value=u'hoffnung', gap=NO_GAP), Token(start_line=8, end_line=8, start_char=63, end_char=66, value=u'daß', gap=NO_GAP), Token(start_line=8, end_line=8, start_char=67, end_char=69, value=u'es', gap=NO_GAP), Token(start_line=8, end_line=8, start_char=70, end_char=75, value=u'ihnen', gap=NO_GAP), Token(start_line=8, end_line=8, start_char=76, end_char=79, value=u'von', gap=NO_GAP), Token(start_line=9, end_line=9, start_char=1, end_char=7, value=u'nutzen', gap=NO_GAP), Token(start_line=9, end_line=9, start_char=9, end_char=10, value=u'n', gap=NO_GAP), Token(start_line=10, end_line=10, start_char=1, end_char=5, value=u'sein', gap=NO_GAP), Token(start_line=10, end_line=10, start_char=6, end_char=10, value=u'wird', gap=NO_GAP), Token(start_line=10, end_line=10, start_char=12, end_char=16, value=u'aber', gap=NO_GAP), Token(start_line=10, end_line=10, start_char=17, end_char=21, value=u'ohne', gap=NO_GAP), Token(start_line=10, end_line=10, start_char=22, end_char=32, value=u'irgendeine', gap=NO_GAP), Token(start_line=10, end_line=10, start_char=33, end_char=41, value=u'garantie', gap=NO_GAP), Token(start_line=10, end_line=10, start_char=43, end_char=48, value=u'sogar', gap=NO_GAP), Token(start_line=10, end_line=10, start_char=49, end_char=53, value=u'ohne', gap=NO_GAP), Token(start_line=10, end_line=10, start_char=54, end_char=57, value=u'die', gap=NO_GAP), Token(start_line=10, end_line=10, start_char=58, end_char=67, value=u'implizite', gap=NO_GAP), Token(start_line=10, end_line=10, start_char=68, end_char=76, value=u'garantie', gap=NO_GAP), Token(start_line=11, end_line=11, start_char=1, end_char=4, value=u'der', gap=NO_GAP), Token(start_line=11, end_line=11, start_char=5, end_char=15, value=u'marktreife', gap=NO_GAP), Token(start_line=11, end_line=11, start_char=17, end_char=18, value=u'n', gap=NO_GAP), Token(start_line=12, end_line=12, start_char=1, end_char=5, value=u'oder', gap=NO_GAP), Token(start_line=12, end_line=12, start_char=6, end_char=9, value=u'der', gap=NO_GAP), Token(start_line=12, end_line=12, start_char=10, end_char=24, value=u'verwendbarkeit', gap=NO_GAP), Token(start_line=12, end_line=12, start_char=25, end_char=28, value=u'für', gap=NO_GAP), Token(start_line=12, end_line=12, start_char=29, end_char=34, value=u'einen', gap=NO_GAP), Token(start_line=12, end_line=12, start_char=35, end_char=45, value=u'bestimmten', gap=NO_GAP), Token(start_line=12, end_line=12, start_char=46, end_char=51, value=u'zweck', gap=NO_GAP), Token(start_line=12, end_line=12, start_char=53, end_char=60, value=u'details', gap=NO_GAP), Token(start_line=12, end_line=12, start_char=61, end_char=67, value=u'finden', gap=NO_GAP), Token(start_line=12, end_line=12, start_char=68, end_char=71, value=u'sie', gap=NO_GAP), Token(start_line=12, end_line=12, start_char=72, end_char=74, value=u'in', gap=NO_GAP), Token(start_line=13, end_line=13, start_char=1, end_char=4, value=u'der', gap=NO_GAP), Token(start_line=13, end_line=13, start_char=5, end_char=8, value=u'gnu', gap=NO_GAP), Token(start_line=13, end_line=13, start_char=9, end_char=16, value=u'general', gap=NO_GAP), Token(start_line=13, end_line=13, start_char=18, end_char=19, value=u'n', gap=NO_GAP), Token(start_line=14, end_line=14, start_char=1, end_char=7, value=u'public', gap=NO_GAP), Token(start_line=14, end_line=14, start_char=8, end_char=15, value=u'license', gap=NO_GAP), Token(start_line=14, end_line=14, start_char=17, end_char=18, value=u'n', gap=NO_GAP), Token(start_line=15, end_line=15, start_char=2, end_char=3, value=u'n', gap=NO_GAP), Token(start_line=16, end_line=16, start_char=1, end_char=4, value=u'sie', gap=NO_GAP), Token(start_line=16, end_line=16, start_char=5, end_char=12, value=u'sollten', gap=NO_GAP), Token(start_line=16, end_line=16, start_char=13, end_char=16, value=u'ein', gap=NO_GAP), Token(start_line=16, end_line=16, start_char=17, end_char=25, value=u'exemplar', gap=NO_GAP), Token(start_line=16, end_line=16, start_char=26, end_char=29, value=u'der', gap=NO_GAP), Token(start_line=16, end_line=16, start_char=30, end_char=33, value=u'gnu', gap=NO_GAP), Token(start_line=16, end_line=16, start_char=34, end_char=41, value=u'general', gap=NO_GAP), Token(start_line=16, end_line=16, start_char=42, end_char=48, value=u'public', gap=NO_GAP), Token(start_line=16, end_line=16, start_char=49, end_char=56, value=u'license', gap=NO_GAP), Token(start_line=16, end_line=16, start_char=57, end_char=65, value=u'zusammen', gap=NO_GAP), Token(start_line=16, end_line=16, start_char=66, end_char=69, value=u'mit', gap=DEFAULT_GAP), Token(start_line=17, end_line=17, start_char=2, end_char=3, value=u'n', gap=NO_GAP), Token(start_line=18, end_line=18, start_char=1, end_char=9, value=u'erhalten', gap=NO_GAP), Token(start_line=18, end_line=18, start_char=10, end_char=15, value=u'haben', gap=NO_GAP), Token(start_line=18, end_line=18, start_char=17, end_char=22, value=u'falls', gap=NO_GAP), Token(start_line=18, end_line=18, start_char=23, end_char=28, value=u'nicht', gap=NO_GAP), Token(start_line=18, end_line=18, start_char=30, end_char=39, value=u'schreiben', gap=NO_GAP), Token(start_line=18, end_line=18, start_char=40, end_char=43, value=u'sie', gap=NO_GAP), Token(start_line=18, end_line=18, start_char=44, end_char=46, value=u'an', gap=NO_GAP), Token(start_line=18, end_line=18, start_char=47, end_char=50, value=u'die', gap=NO_GAP), Token(start_line=18, end_line=18, start_char=51, end_char=55, value=u'free', gap=NO_GAP), Token(start_line=18, end_line=18, start_char=56, end_char=64, value=u'software', gap=NO_GAP), Token(start_line=18, end_line=18, start_char=65, end_char=75, value=u'foundation', gap=NO_GAP), Token(start_line=19, end_line=19, start_char=2, end_char=3, value=u'n', gap=NO_GAP), Token(start_line=20, end_line=20, start_char=1, end_char=4, value=u'inc', gap=NO_GAP), Token(start_line=20, end_line=20, start_char=7, end_char=9, value=u'51', gap=NO_GAP), Token(start_line=20, end_line=20, start_char=10, end_char=18, value=u'franklin', gap=NO_GAP), Token(start_line=20, end_line=20, start_char=19, end_char=21, value=u'st', gap=NO_GAP), Token(start_line=20, end_line=20, start_char=23, end_char=28, value=u'fifth', gap=NO_GAP), Token(start_line=20, end_line=20, start_char=29, end_char=34, value=u'floor', gap=NO_GAP), Token(start_line=20, end_line=20, start_char=36, end_char=42, value=u'boston', gap=NO_GAP), Token(start_line=20, end_line=20, start_char=44, end_char=46, value=u'ma', gap=NO_GAP), Token(start_line=20, end_line=20, start_char=47, end_char=52, value=u'02110', gap=NO_GAP), Token(start_line=20, end_line=20, start_char=54, end_char=57, value=u'usa', gap=NO_GAP)]
    test_file = self.get_test_loc('analysis/unicode/12180.txt')
    with codecs.open(test_file, encoding='utf-8') as test:
        result = list(self.template_parsing(test))
        assert (expected == result)