# Path: JapaneseTokenizer/common/text_preprocess.py
# def normalize_text(input_text,
#                    dictionary_mode='ipadic',
#                    new_line_replaced='。',
#                    is_replace_eos=True,
#                    is_kana=True,
#                    is_ascii=True,
#                    is_digit=True):
#     # type: (text_type,text_type,text_type,bool,bool,bool,bool)->text_type
#     """* What you can do
#     - It converts input-text into normalized-text which is good for tokenizer input.
# 
#     * Params
#     - new_line_replaced: a string which replaces from \n string.
#     """
#     if is_replace_eos:
#         without_new_line = input_text.replace('\n', new_line_replaced)
#     else:
#         without_new_line = new_line_replaced
# 
#     if dictionary_mode=='neologd' and is_neologdn_valid:
#         return neologdn.normalize(normalize_text_normal_ipadic(without_new_line))
#     elif dictionary_mode=='neologd' and is_neologdn_valid == False:
#         raise Exception("You could not call neologd dictionary bacause you do NOT install the package neologdn.")
#     else:
#         return normalize_text_normal_ipadic(without_new_line, kana=is_kana, ascii=is_ascii, digit=is_digit)
# 
# def denormalize_text(input_text):
#     # type: (text_type)->text_type
#     """* What you can do
#     - It converts text into standard japanese writing way
# 
#     * Note
#     - hankaku-katakana is to zenkaku-katakana
#     - zenkaku-eisu is to hankaku-eisu
#     """
#     if input_text in STRING_EXCEPTION:
#         return input_text
#     else:
#         return jaconv.z2h(input_text, kana=False, ascii=True, digit=True)


from JapaneseTokenizer.common.text_preprocess import normalize_text, denormalize_text
from MeCab import Node
from typing import List, Union, Any, Tuple, Dict, Callable, Optional
from future.utils import text_type, string_types
import sys
import six
        self.is_feature = is_feature
        self.misc_info = misc_info
        self.analyzed_line = analyzed_line

        if isinstance(tuple_pos, tuple):
            self.tuple_pos = tuple_pos
        elif isinstance(tuple_pos, string_types):
            self.tuple_pos = ('*', )
        else:
            raise Exception('Error while parsing feature object. {}'.format(tuple_pos))


class TokenizedSenetence(object):
    def __init__(self, sentence, tokenized_objects, string_encoding='utf-8'):
        # type: (text_type, List[TokenizedResult], text_type)->None
        """* Parameters
        - sentence: sentence
        - tokenized_objects: list of TokenizedResult object
        - string_encoding: Encoding type of string type. This option is used only under python2.x
        """
        assert isinstance(sentence, text_type)
        assert isinstance(tokenized_objects, list)

        self.sentence = sentence
        self.tokenized_objects = tokenized_objects
        self.string_encoding = string_encoding


    def __extend_token_object(self, token_object,
                              is_denormalize=True,

# Path: JapaneseTokenizer/common/text_preprocess.py
# def normalize_text(input_text,
#                    dictionary_mode='ipadic',
#                    new_line_replaced='。',
#                    is_replace_eos=True,
#                    is_kana=True,
#                    is_ascii=True,
#                    is_digit=True):
#     # type: (text_type,text_type,text_type,bool,bool,bool,bool)->text_type
#     """* What you can do
#     - It converts input-text into normalized-text which is good for tokenizer input.
# 
#     * Params
#     - new_line_replaced: a string which replaces from \n string.
#     """
#     if is_replace_eos:
#         without_new_line = input_text.replace('\n', new_line_replaced)
#     else:
#         without_new_line = new_line_replaced
# 
#     if dictionary_mode=='neologd' and is_neologdn_valid:
#         return neologdn.normalize(normalize_text_normal_ipadic(without_new_line))
#     elif dictionary_mode=='neologd' and is_neologdn_valid == False:
#         raise Exception("You could not call neologd dictionary bacause you do NOT install the package neologdn.")
#     else:
#         return normalize_text_normal_ipadic(without_new_line, kana=is_kana, ascii=is_ascii, digit=is_digit)
# 
# def denormalize_text(input_text):
#     # type: (text_type)->text_type
#     """* What you can do
#     - It converts text into standard japanese writing way
# 
#     * Note
#     - hankaku-katakana is to zenkaku-katakana
#     - zenkaku-eisu is to hankaku-eisu
#     """
#     if input_text in STRING_EXCEPTION:
#         return input_text
#     else:
#         return jaconv.z2h(input_text, kana=False, ascii=True, digit=True)


# Path: JapaneseTokenizer/datamodels.py
from JapaneseTokenizer.common.text_preprocess import normalize_text, denormalize_text
from MeCab import Node
from typing import List, Union, Any, Tuple, Dict, Callable, Optional
from future.utils import text_type, string_types
import sys
import six

        self.is_feature = is_feature
        self.misc_info = misc_info
        self.analyzed_line = analyzed_line

        if isinstance(tuple_pos, tuple):
            self.tuple_pos = tuple_pos
        elif isinstance(tuple_pos, string_types):
            self.tuple_pos = ('*', )
        else:
            raise Exception('Error while parsing feature object. {}'.format(tuple_pos))


class TokenizedSenetence(object):
    def __init__(self, sentence, tokenized_objects, string_encoding='utf-8'):
        # type: (text_type, List[TokenizedResult], text_type)->None
        """* Parameters
        - sentence: sentence
        - tokenized_objects: list of TokenizedResult object
        - string_encoding: Encoding type of string type. This option is used only under python2.x
        """
        assert isinstance(sentence, text_type)
        assert isinstance(tokenized_objects, list)

        self.sentence = sentence
        self.tokenized_objects = tokenized_objects
        self.string_encoding = string_encoding


    def __extend_token_object(self, token_object,
                              is_denormalize=True,

                              func_denormalizer=denormalize_text):