<commit_before># -*- coding: utf-8 -*-


import sys
from .domecab import do_mecab, do_mecab_iter


class Token:
    """
    Token type
    """

    def __init__(self, **kwargs):
        # 表層形\t品詞,品詞細分類1,品詞細分類2,品詞細分類3,活用型,活用形,原形,読み,発音
        self.surface     = kwargs.pop('surface'  , None)
        self.pos         = kwargs.pop('pos'      , None)
        self.pos_detail1 = kwargs.pop('detail1'  , None)
        self.pos_detail2 = kwargs.pop('detail2'  , None)
        self.pos_detail3 = kwargs.pop('detail3'  , None)
        self.infl_type   = kwargs.pop('infl_type', None)
        self.infl_form   = kwargs.pop('infl_form', None)
        self.base_form   = kwargs.pop('base_form', None)
        self.reading     = kwargs.pop('reading'  , None)
        self.phoenetic   = kwargs.pop('phoenetic', None)


    def __str__(self):
        tmp = [
            self.surface, 
            self.pos,
            self.pos_detail1,
            self.pos_detail2,
            self.infl_type,
            self.infl_form,
            self.base_form,
            self.reading,
            self.phoenetic
        ]
        if sys.version_info[0] == 2:
            tmp = ['*' if a is None else a for a in tmp]
            tmp = tuple(a.encode('utf8') for a in tmp)
        else:
            tmp = tuple('*' if a is None else a for a in tmp)
        
        out = '%s\t%s,%s,%s,%s,%s,%s,%s,%s' % tmp
        return out
    
    
    def __format__(self, formatspec):
        """
        encode token info in an arbitrary encoding
        implemented for python 2 with 
        encoding other than utf8
        
        :param formatspec:  encoding
        """
        tmp = [
            self.surface, 
            self.pos,
            self.pos_detail1,
            self.pos_detail2,
            self.infl_type,
            self.infl_form,
            self.base_form,
            self.reading,
            self.phoenetic
        ]
        if sys.version_info[0] == 2:
            tmp = ['*' if a is None else a for a in tmp]
            tmp = tuple(a.encode(formatspec) for a in tmp)
        else:
            tmp = tuple('*' if a is None else a for a in tmp)
        
        out = '%s\t%s,%s,%s,%s,%s,%s,%s,%s' % tmp
        return out
        

def tokenize(x, mecab_enc='utf8'):
    """
    Tokenize a string 

    :param x:   string

    :return:    generator of tokens
    """

    o = do_mecab(x, mecab_enc=mecab_enc).split('\n')
    for line in o:
        if line.strip()=="EOS":
            break

        surface, info = line.split("\t")
        info = info.split(",")

        pos         = info[0] if len(info) > 0 else None
        pos_detail1 = info[1] if len(info) > 1 else None
        pos_detail2 = info[2] if len(info) > 2 else None
        pos_detail3 = info[3] if len(info) > 3 else None
        infl_type   = info[4] if len(info) > 4 else None
        infl_form   = info[5] if len(info) > 5 else None
        base_form   = info[6] if len(info) > 6 else None
        reading     = info[7] if len(info) > 7 else None
        phoenetic   = info[8] if len(info) > 8 else None<commit_msg>Update api.py<commit_after># -*- coding: utf-8 -*-


import sys
from .domecab import do_mecab, do_mecab_iter


class Token:
    """
    Token type
    """

    def __init__(self, **kwargs):
        self.surface     = kwargs.pop('surface'  , None)
        self.pos         = kwargs.pop('pos'      , None)
        self.pos_detail1 = kwargs.pop('detail1'  , None)
        self.pos_detail2 = kwargs.pop('detail2'  , None)
        self.pos_detail3 = kwargs.pop('detail3'  , None)
        self.infl_type   = kwargs.pop('infl_type', None)
        self.infl_form   = kwargs.pop('infl_form', None)
        self.base_form   = kwargs.pop('base_form', None)
        self.reading     = kwargs.pop('reading'  , None)
        self.phoenetic   = kwargs.pop('phoenetic', None)


    def __str__(self):
        tmp = [
            self.surface, 
            self.pos,
            self.pos_detail1,
            self.pos_detail2,
            self.infl_type,
            self.infl_form,
            self.base_form,
            self.reading,
            self.phoenetic
        ]
        if sys.version_info[0] == 2:
            tmp = ['*' if a is None else a for a in tmp]
            tmp = tuple(a.encode('utf8') for a in tmp)
        else:
            tmp = tuple('*' if a is None else a for a in tmp)
        
        out = '%s\t%s,%s,%s,%s,%s,%s,%s,%s' % tmp
        return out
    
    
    def __format__(self, formatspec):
        """
        encode token info in an arbitrary encoding
        implemented for python 2 with 
        encoding other than utf8
        
        :param formatspec:  encoding
        """
        tmp = [
            self.surface, 
            self.pos,
            self.pos_detail1,
            self.pos_detail2,
            self.infl_type,
            self.infl_form,
            self.base_form,
            self.reading,
            self.phoenetic
        ]
        if sys.version_info[0] == 2:
            tmp = ['*' if a is None else a for a in tmp]
            tmp = tuple(a.encode(formatspec) for a in tmp)
        else:
            tmp = tuple('*' if a is None else a for a in tmp)
        
        out = '%s\t%s,%s,%s,%s,%s,%s,%s,%s' % tmp
        return out
        

def tokenize(x, mecab_enc='utf8'):
    """
    Tokenize a string 

    :param x:   string

    :return:    generator of tokens
    """

    o = do_mecab(x, mecab_enc=mecab_enc).split('\n')
    for line in o:
        if line.strip()=="EOS":
            break
            
        # 表層形\t品詞,品詞細分類1,品詞細分類2,品詞細分類3,活用型,活用形,原形,読み,発音
        surface, info = line.split("\t")
        info = info.split(",")

        pos         = info[0] if len(info) > 0 else None
        pos_detail1 = info[1] if len(info) > 1 else None
        pos_detail2 = info[2] if len(info) > 2 else None
        pos_detail3 = info[3] if len(info) > 3 else None
        infl_type   = info[4] if len(info) > 4 else None
        infl_form   = info[5] if len(info) > 5 else None
        base_form   = info[6] if len(info) > 6 else None
        reading     = info[7] if len(info) > 7 else None
        phoenetic   = info[8] if len(info) > 8 else None
{'fraction_non_alphanumeric': 0.12031653746770025, 'fraction_numerical': 0.013888888888888888, 'mean_word_length': 1.5988250104909778, 'pattern_counts': {'":': 2, '<': 3, '<?xml version=': 0, '>': 21, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1305788', 'n_tokens_mistral': 2161, 'n_tokens_neox': 2121, 'n_words': 518}