# Path: JapaneseTokenizer/common/text_preprocess.py
# def normalize_text(input_text,
#                    dictionary_mode='ipadic',
#                    new_line_replaced='。',
#                    is_replace_eos=True,
#                    is_kana=True,
#                    is_ascii=True,
#                    is_digit=True):
#     # type: (text_type,text_type,text_type,bool,bool,bool,bool)->text_type
#     """* What you can do
#     - It converts input-text into normalized-text which is good for tokenizer input.
# 
#     * Params
#     - new_line_replaced: a string which replaces from \n string.
#     """
#     if is_replace_eos:
#         without_new_line = input_text.replace('\n', new_line_replaced)
#     else:
#         without_new_line = new_line_replaced
# 
#     if dictionary_mode=='neologd' and is_neologdn_valid:
#         return neologdn.normalize(normalize_text_normal_ipadic(without_new_line))
#     elif dictionary_mode=='neologd' and is_neologdn_valid == False:
#         raise Exception("You could not call neologd dictionary bacause you do NOT install the package neologdn.")
#     else:
#         return normalize_text_normal_ipadic(without_new_line, kana=is_kana, ascii=is_ascii, digit=is_digit)
# 
# def denormalize_text(input_text):
#     # type: (text_type)->text_type
#     """* What you can do
#     - It converts text into standard japanese writing way
# 
#     * Note
#     - hankaku-katakana is to zenkaku-katakana
#     - zenkaku-eisu is to hankaku-eisu
#     """
#     if input_text in STRING_EXCEPTION:
#         return input_text
#     else:
#         return jaconv.z2h(input_text, kana=False, ascii=True, digit=True)


from JapaneseTokenizer.common.text_preprocess import normalize_text, denormalize_text
from MeCab import Node
from typing import List, Union, Any, Tuple, Dict, Callable, Optional
from future.utils import text_type, string_types
import sys
import six
        """
        assert isinstance(token_object, TokenizedResult)

        if is_denormalize:
            if token_object.is_feature == True:
                if token_object.is_surface == True:
                    token = (func_denormalizer(token_object.word_surface), token_object.tuple_pos)
                else:
                    token = (func_denormalizer(token_object.word_stem), token_object.tuple_pos)
            else:
                if token_object.is_surface == True:
                    token = func_denormalizer(token_object.word_surface)
                else:
                    token = func_denormalizer(token_object.word_stem)
        else:
            if token_object.is_feature == True:
                if token_object.is_surface == True:
                    token = (token_object.word_surface, token_object.tuple_pos)
                else:
                    token = (token_object.word_stem, token_object.tuple_pos)
            else:
                if token_object.is_surface == True:
                    token = token_object.word_surface
                else:
                    token = token_object.word_stem

        return token

    def convert_list_object(self,
                            is_denormalize=True,

# Path: JapaneseTokenizer/common/text_preprocess.py
# def normalize_text(input_text,
#                    dictionary_mode='ipadic',
#                    new_line_replaced='。',
#                    is_replace_eos=True,
#                    is_kana=True,
#                    is_ascii=True,
#                    is_digit=True):
#     # type: (text_type,text_type,text_type,bool,bool,bool,bool)->text_type
#     """* What you can do
#     - It converts input-text into normalized-text which is good for tokenizer input.
# 
#     * Params
#     - new_line_replaced: a string which replaces from \n string.
#     """
#     if is_replace_eos:
#         without_new_line = input_text.replace('\n', new_line_replaced)
#     else:
#         without_new_line = new_line_replaced
# 
#     if dictionary_mode=='neologd' and is_neologdn_valid:
#         return neologdn.normalize(normalize_text_normal_ipadic(without_new_line))
#     elif dictionary_mode=='neologd' and is_neologdn_valid == False:
#         raise Exception("You could not call neologd dictionary bacause you do NOT install the package neologdn.")
#     else:
#         return normalize_text_normal_ipadic(without_new_line, kana=is_kana, ascii=is_ascii, digit=is_digit)
# 
# def denormalize_text(input_text):
#     # type: (text_type)->text_type
#     """* What you can do
#     - It converts text into standard japanese writing way
# 
#     * Note
#     - hankaku-katakana is to zenkaku-katakana
#     - zenkaku-eisu is to hankaku-eisu
#     """
#     if input_text in STRING_EXCEPTION:
#         return input_text
#     else:
#         return jaconv.z2h(input_text, kana=False, ascii=True, digit=True)


# Path: JapaneseTokenizer/datamodels.py
from JapaneseTokenizer.common.text_preprocess import normalize_text, denormalize_text
from MeCab import Node
from typing import List, Union, Any, Tuple, Dict, Callable, Optional
from future.utils import text_type, string_types
import sys
import six

        """
        assert isinstance(token_object, TokenizedResult)

        if is_denormalize:
            if token_object.is_feature == True:
                if token_object.is_surface == True:
                    token = (func_denormalizer(token_object.word_surface), token_object.tuple_pos)
                else:
                    token = (func_denormalizer(token_object.word_stem), token_object.tuple_pos)
            else:
                if token_object.is_surface == True:
                    token = func_denormalizer(token_object.word_surface)
                else:
                    token = func_denormalizer(token_object.word_stem)
        else:
            if token_object.is_feature == True:
                if token_object.is_surface == True:
                    token = (token_object.word_surface, token_object.tuple_pos)
                else:
                    token = (token_object.word_stem, token_object.tuple_pos)
            else:
                if token_object.is_surface == True:
                    token = token_object.word_surface
                else:
                    token = token_object.word_stem

        return token

    def convert_list_object(self,
                            is_denormalize=True,

                            func_denormalizer=denormalize_text):