# Path: janome/tokenizer.py
# class Token(object):
#     """
#     A Token object contains all information for a token.
#     """
# 
#     def __init__(self, node: Node, extra: Tuple = None):
#         self.node = node
#         self.extra = extra
# 
#     def __getattr__(self, name) -> Any:
#         if name == 'surface':
#             return self.node.surface
#         elif name == 'part_of_speech':
#             return self.extra[0] if self.extra else self.node.part_of_speech
#         elif name == 'infl_type':
#             return self.extra[1] if self.extra else self.node.infl_type
#         elif name == 'infl_form':
#             return self.extra[2] if self.extra else self.node.infl_form
#         elif name == 'base_form':
#             return self.extra[3] if self.extra else self.node.base_form
#         elif name == 'reading':
#             return self.extra[4] if self.extra else self.node.reading
#         elif name == 'phonetic':
#             return self.extra[5] if self.extra else self.node.phonetic
#         elif name == 'node_type':
#             return self.node.node_type
#         else:
#             None
# 
#     def __setattr__(self, name: str, value: Any) -> None:
#         # see https://stackoverflow.com/questions/61213745/typechecking-dynamically-added-attributes
#         object.__setattr__(self, name, value)
# 
#     def __str__(self):
#         return f'{self.surface}\t' \
#             f'{self.part_of_speech},{self.infl_type},{self.infl_form},{self.base_form},{self.reading},{self.phonetic}'


from abc import ABC, abstractmethod
from collections import defaultdict
from typing import Iterator, List, Dict, Tuple, Any
from .tokenizer import Token

    Added in *version 0.3.4*
    """

    def apply(self, tokens: Iterator[Token]) -> Iterator[Token]:
        for token in tokens:
            token.surface = token.surface.upper()
            token.base_form = token.base_form.upper()
            yield token


class POSStopFilter(TokenFilter):
    u"""
    A POSStopFilter removes tokens associated with part-of-speech tags
    listed in the stop tags list and keeps other tokens.

    Tag matching rule is prefix-matching. e.g., if '動詞' is given as a stop tag,
    '動詞,自立,*,*' and '動詞,非自立,*,*' (or so) are removed.

    Added in *version 0.3.4*
    """

    def __init__(self, pos_list: List[str]):
        """
        Initialize POSStopFilter object.

        :param pos_list: stop part-of-speech tags list.
        """
        self.pos_list = pos_list


# Path: janome/tokenizer.py
# class Token(object):
#     """
#     A Token object contains all information for a token.
#     """
# 
#     def __init__(self, node: Node, extra: Tuple = None):
#         self.node = node
#         self.extra = extra
# 
#     def __getattr__(self, name) -> Any:
#         if name == 'surface':
#             return self.node.surface
#         elif name == 'part_of_speech':
#             return self.extra[0] if self.extra else self.node.part_of_speech
#         elif name == 'infl_type':
#             return self.extra[1] if self.extra else self.node.infl_type
#         elif name == 'infl_form':
#             return self.extra[2] if self.extra else self.node.infl_form
#         elif name == 'base_form':
#             return self.extra[3] if self.extra else self.node.base_form
#         elif name == 'reading':
#             return self.extra[4] if self.extra else self.node.reading
#         elif name == 'phonetic':
#             return self.extra[5] if self.extra else self.node.phonetic
#         elif name == 'node_type':
#             return self.node.node_type
#         else:
#             None
# 
#     def __setattr__(self, name: str, value: Any) -> None:
#         # see https://stackoverflow.com/questions/61213745/typechecking-dynamically-added-attributes
#         object.__setattr__(self, name, value)
# 
#     def __str__(self):
#         return f'{self.surface}\t' \
#             f'{self.part_of_speech},{self.infl_type},{self.infl_form},{self.base_form},{self.reading},{self.phonetic}'


# Path: janome/tokenfilter.py
from abc import ABC, abstractmethod
from collections import defaultdict
from typing import Iterator, List, Dict, Tuple, Any
from .tokenizer import Token


    Added in *version 0.3.4*
    """

    def apply(self, tokens: Iterator[Token]) -> Iterator[Token]:
        for token in tokens:
            token.surface = token.surface.upper()
            token.base_form = token.base_form.upper()
            yield token


class POSStopFilter(TokenFilter):
    u"""
    A POSStopFilter removes tokens associated with part-of-speech tags
    listed in the stop tags list and keeps other tokens.

    Tag matching rule is prefix-matching. e.g., if '動詞' is given as a stop tag,
    '動詞,自立,*,*' and '動詞,非自立,*,*' (or so) are removed.

    Added in *version 0.3.4*
    """

    def __init__(self, pos_list: List[str]):
        """
        Initialize POSStopFilter object.

        :param pos_list: stop part-of-speech tags list.
        """
        self.pos_list = pos_list


    def apply(self, tokens: Iterator[Token]) -> Iterator[Token]: