# Path: neuronlp2/io/instance.py
# class DependencyInstance(object):
#     def __init__(self, sentence, postags, pos_ids, heads, types, type_ids):
#         self.sentence = sentence
#         self.postags = postags
#         self.pos_ids = pos_ids
#         self.heads = heads
#         self.types = types
#         self.type_ids = type_ids
# 
#     def length(self):
#         return self.sentence.length()
# 
# class NERInstance(object):
#     def __init__(self, sentence, postags, pos_ids, chunk_tags, chunk_ids, ner_tags, ner_ids):
#         self.sentence = sentence
#         self.postags = postags
#         self.pos_ids = pos_ids
#         self.chunk_tags = chunk_tags
#         self.chunk_ids = chunk_ids
#         self.ner_tags = ner_tags
#         self.ner_ids = ner_ids
# 
#     def length(self):
#         return self.sentence.length()
# 
# Path: neuronlp2/io/instance.py
# class Sentence(object):
#     def __init__(self, words, word_ids, char_seqs, char_id_seqs):
#         self.words = words
#         self.word_ids = word_ids
#         self.char_seqs = char_seqs
#         self.char_id_seqs = char_id_seqs
# 
#     def length(self):
#         return len(self.words)
# 
# Path: neuronlp2/io/common.py
# ROOT = "_ROOT"
# 
# ROOT_POS = "_ROOT_POS"
# 
# ROOT_CHAR = "_ROOT_CHAR"
# 
# ROOT_TYPE = "_<ROOT>"
# 
# END = "_END"
# 
# END_POS = "_END_POS"
# 
# END_CHAR = "_END_CHAR"
# 
# END_TYPE = "_<END>"
# 
# Path: neuronlp2/io/common.py
# DIGIT_RE = re.compile(r"\d")
# 
# MAX_CHAR_LENGTH = 45


from neuronlp2.io.instance import DependencyInstance, NERInstance
from neuronlp2.io.instance import Sentence
from neuronlp2.io.common import ROOT, ROOT_POS, ROOT_CHAR, ROOT_TYPE, END, END_POS, END_CHAR, END_TYPE
from neuronlp2.io.common import DIGIT_RE, MAX_CHAR_LENGTH
            char_ids = []
            for char in tokens[1]:
                chars.append(char)
                char_ids.append(self.__char_alphabet.get_index(char))
            if len(chars) > MAX_CHAR_LENGTH:
                chars = chars[:MAX_CHAR_LENGTH]
                char_ids = char_ids[:MAX_CHAR_LENGTH]
            char_seqs.append(chars)
            char_id_seqs.append(char_ids)

            word = DIGIT_RE.sub("0", tokens[1]) if normalize_digits else tokens[1]
            pos = tokens[4]
            head = int(tokens[6])
            type = tokens[7]

            words.append(word)
            word_ids.append(self.__word_alphabet.get_index(word))

            postags.append(pos)
            pos_ids.append(self.__pos_alphabet.get_index(pos))

            types.append(type)
            type_ids.append(self.__type_alphabet.get_index(type))

            heads.append(head)

        if symbolic_end:
            words.append(END)
            word_ids.append(self.__word_alphabet.get_index(END))
            char_seqs.append([END_CHAR, ])

# Path: neuronlp2/io/instance.py
# class DependencyInstance(object):
#     def __init__(self, sentence, postags, pos_ids, heads, types, type_ids):
#         self.sentence = sentence
#         self.postags = postags
#         self.pos_ids = pos_ids
#         self.heads = heads
#         self.types = types
#         self.type_ids = type_ids
# 
#     def length(self):
#         return self.sentence.length()
# 
# class NERInstance(object):
#     def __init__(self, sentence, postags, pos_ids, chunk_tags, chunk_ids, ner_tags, ner_ids):
#         self.sentence = sentence
#         self.postags = postags
#         self.pos_ids = pos_ids
#         self.chunk_tags = chunk_tags
#         self.chunk_ids = chunk_ids
#         self.ner_tags = ner_tags
#         self.ner_ids = ner_ids
# 
#     def length(self):
#         return self.sentence.length()
# 
# Path: neuronlp2/io/instance.py
# class Sentence(object):
#     def __init__(self, words, word_ids, char_seqs, char_id_seqs):
#         self.words = words
#         self.word_ids = word_ids
#         self.char_seqs = char_seqs
#         self.char_id_seqs = char_id_seqs
# 
#     def length(self):
#         return len(self.words)
# 
# Path: neuronlp2/io/common.py
# ROOT = "_ROOT"
# 
# ROOT_POS = "_ROOT_POS"
# 
# ROOT_CHAR = "_ROOT_CHAR"
# 
# ROOT_TYPE = "_<ROOT>"
# 
# END = "_END"
# 
# END_POS = "_END_POS"
# 
# END_CHAR = "_END_CHAR"
# 
# END_TYPE = "_<END>"
# 
# Path: neuronlp2/io/common.py
# DIGIT_RE = re.compile(r"\d")
# 
# MAX_CHAR_LENGTH = 45


# Path: neuronlp2/io/reader.py
from neuronlp2.io.instance import DependencyInstance, NERInstance
from neuronlp2.io.instance import Sentence
from neuronlp2.io.common import ROOT, ROOT_POS, ROOT_CHAR, ROOT_TYPE, END, END_POS, END_CHAR, END_TYPE
from neuronlp2.io.common import DIGIT_RE, MAX_CHAR_LENGTH

            char_ids = []
            for char in tokens[1]:
                chars.append(char)
                char_ids.append(self.__char_alphabet.get_index(char))
            if len(chars) > MAX_CHAR_LENGTH:
                chars = chars[:MAX_CHAR_LENGTH]
                char_ids = char_ids[:MAX_CHAR_LENGTH]
            char_seqs.append(chars)
            char_id_seqs.append(char_ids)

            word = DIGIT_RE.sub("0", tokens[1]) if normalize_digits else tokens[1]
            pos = tokens[4]
            head = int(tokens[6])
            type = tokens[7]

            words.append(word)
            word_ids.append(self.__word_alphabet.get_index(word))

            postags.append(pos)
            pos_ids.append(self.__pos_alphabet.get_index(pos))

            types.append(type)
            type_ids.append(self.__type_alphabet.get_index(type))

            heads.append(head)

        if symbolic_end:
            words.append(END)
            word_ids.append(self.__word_alphabet.get_index(END))
            char_seqs.append([END_CHAR, ])

            char_id_seqs.append([self.__char_alphabet.get_index(END_CHAR), ])