# Path: JapaneseTokenizer/common/text_preprocess.py
# def normalize_text(input_text,
#                    dictionary_mode='ipadic',
#                    new_line_replaced='。',
#                    is_replace_eos=True,
#                    is_kana=True,
#                    is_ascii=True,
#                    is_digit=True):
#     # type: (text_type,text_type,text_type,bool,bool,bool,bool)->text_type
#     """* What you can do
#     - It converts input-text into normalized-text which is good for tokenizer input.
# 
#     * Params
#     - new_line_replaced: a string which replaces from \n string.
#     """
#     if is_replace_eos:
#         without_new_line = input_text.replace('\n', new_line_replaced)
#     else:
#         without_new_line = new_line_replaced
# 
#     if dictionary_mode=='neologd' and is_neologdn_valid:
#         return neologdn.normalize(normalize_text_normal_ipadic(without_new_line))
#     elif dictionary_mode=='neologd' and is_neologdn_valid == False:
#         raise Exception("You could not call neologd dictionary bacause you do NOT install the package neologdn.")
#     else:
#         return normalize_text_normal_ipadic(without_new_line, kana=is_kana, ascii=is_ascii, digit=is_digit)
# 
# def denormalize_text(input_text):
#     # type: (text_type)->text_type
#     """* What you can do
#     - It converts text into standard japanese writing way
# 
#     * Note
#     - hankaku-katakana is to zenkaku-katakana
#     - zenkaku-eisu is to hankaku-eisu
#     """
#     if input_text in STRING_EXCEPTION:
#         return input_text
#     else:
#         return jaconv.z2h(input_text, kana=False, ascii=True, digit=True)


from JapaneseTokenizer.common.text_preprocess import normalize_text, denormalize_text
from MeCab import Node
from typing import List, Union, Any, Tuple, Dict, Callable, Optional
from future.utils import text_type, string_types
import sys
import six
            raise Exception('Pos condition expects tuple of string. However = {}'.format(p_c_tuple))

        converted = [text_type] * len(p_c_tuple)
        for i, pos_element in enumerate(p_c_tuple):
            if six.PY2 and isinstance(pos_element, str):
                """str into unicode if python2.x"""
                converted[i] = pos_element.decode(self.string_encoding)
            elif six.PY2 and isinstance(pos_element, text_type):
                converted[i] = pos_element
            elif six.PY3:
                converted[i] = pos_element
            else:
                raise Exception()

        return tuple(converted)

    def __check_pos_condition(self, pos_condistion):
        # type: (List[Tuple[text_type, ...]])->List[Tuple[text_type, ...]]
        """* What you can do
        - Check your pos condition
        - It converts character type into unicode if python version is 2.x
        """
        assert isinstance(pos_condistion, list)

        return [self.__convert_string_type(p_c_tuple) for p_c_tuple in pos_condistion]

    def filter(self,
               pos_condition=None,
               stopwords=None,
               is_normalize=True,

# Path: JapaneseTokenizer/common/text_preprocess.py
# def normalize_text(input_text,
#                    dictionary_mode='ipadic',
#                    new_line_replaced='。',
#                    is_replace_eos=True,
#                    is_kana=True,
#                    is_ascii=True,
#                    is_digit=True):
#     # type: (text_type,text_type,text_type,bool,bool,bool,bool)->text_type
#     """* What you can do
#     - It converts input-text into normalized-text which is good for tokenizer input.
# 
#     * Params
#     - new_line_replaced: a string which replaces from \n string.
#     """
#     if is_replace_eos:
#         without_new_line = input_text.replace('\n', new_line_replaced)
#     else:
#         without_new_line = new_line_replaced
# 
#     if dictionary_mode=='neologd' and is_neologdn_valid:
#         return neologdn.normalize(normalize_text_normal_ipadic(without_new_line))
#     elif dictionary_mode=='neologd' and is_neologdn_valid == False:
#         raise Exception("You could not call neologd dictionary bacause you do NOT install the package neologdn.")
#     else:
#         return normalize_text_normal_ipadic(without_new_line, kana=is_kana, ascii=is_ascii, digit=is_digit)
# 
# def denormalize_text(input_text):
#     # type: (text_type)->text_type
#     """* What you can do
#     - It converts text into standard japanese writing way
# 
#     * Note
#     - hankaku-katakana is to zenkaku-katakana
#     - zenkaku-eisu is to hankaku-eisu
#     """
#     if input_text in STRING_EXCEPTION:
#         return input_text
#     else:
#         return jaconv.z2h(input_text, kana=False, ascii=True, digit=True)


# Path: JapaneseTokenizer/datamodels.py
from JapaneseTokenizer.common.text_preprocess import normalize_text, denormalize_text
from MeCab import Node
from typing import List, Union, Any, Tuple, Dict, Callable, Optional
from future.utils import text_type, string_types
import sys
import six

            raise Exception('Pos condition expects tuple of string. However = {}'.format(p_c_tuple))

        converted = [text_type] * len(p_c_tuple)
        for i, pos_element in enumerate(p_c_tuple):
            if six.PY2 and isinstance(pos_element, str):
                """str into unicode if python2.x"""
                converted[i] = pos_element.decode(self.string_encoding)
            elif six.PY2 and isinstance(pos_element, text_type):
                converted[i] = pos_element
            elif six.PY3:
                converted[i] = pos_element
            else:
                raise Exception()

        return tuple(converted)

    def __check_pos_condition(self, pos_condistion):
        # type: (List[Tuple[text_type, ...]])->List[Tuple[text_type, ...]]
        """* What you can do
        - Check your pos condition
        - It converts character type into unicode if python version is 2.x
        """
        assert isinstance(pos_condistion, list)

        return [self.__convert_string_type(p_c_tuple) for p_c_tuple in pos_condistion]

    def filter(self,
               pos_condition=None,
               stopwords=None,
               is_normalize=True,

               func_normalizer=normalize_text,