<commit_before>#!/usr/bin/env python
# -*- coding: utf-8 -*-

from __future__ import unicode_literals
from argparse import ArgumentParser
import sys

import pypinyin
from pypinyin import __title__, __version__

py3 = sys.version_info[0] == 3
if py3:
    unicode = str


def get_parser():
    parser = ArgumentParser(description='convert chinese to pinyin.')
    parser.add_argument('-V', '--version', action='version',
                        version='{0} {1}'.format(__title__, __version__))
    # 要执行的函数名称
    parser.add_argument('--func', help='function name (default: "pinyin")',
                        choices=['pinyin', 'slug'],
                        default='pinyin')
    # 拼音风格
    parser.add_argument('--style', help='pinyin style (default: "TONE")',
                        choices=['NORMAL', 'TONE', 'TONE2', 'INITIALS',
                                 'FIRST_LETTER', 'FINALS', 'FINALS_TONE',
                                 'FINALS_TONE2'], default='TONE')
    parser.add_argument('--separator', help='slug separator (default: "-")',
                        default='-')
    # 输出多音字
    parser.add_argument('--heteronym', help='enable heteronym',
                        action='store_true')
    # 要查询的汉字
    parser.add_argument('hans', help='chinese string')
    return parser


def main():
    import logging
    # 禁用除 CRITICAL 外的日志消息
    logging.disable(logging.CRITICAL)

    # 获取命令行选项和参数
    parser = get_parser()
    options = parser.parse_args()
    if py3:
        hans = options.hans
    else:
        hans = options.hans.decode(sys.stdin.encoding)
    func = getattr(pypinyin, options.func, 'pinyin')
    style = getattr(pypinyin, options.style, 'TONE')
    heteronym = options.heteronym
    separator = options.separator

    func_kwargs = {
        'pinyin': {'heteronym': heteronym},
        'slug': {'heteronym': heteronym, 'separator': separator},
    }
    if py3:
        kwargs = func_kwargs[func.__name__]
    else:
        kwargs = func_kwargs[func.func_name]
    result = func(hans, style=style, **kwargs)

    if result and isinstance(result, (list, tuple)):
        if isinstance(result[0], (list, tuple)):
            print(' '.join([','.join(s) for s in result]))
        else:
            print(result)
    else:
        print(result)

if __name__ == '__main__':
    main()<commit_msg>update runner.py<commit_after>#!/usr/bin/env python
# -*- coding: utf-8 -*-

from __future__ import unicode_literals
from argparse import ArgumentParser
import sys

import pypinyin
from pypinyin import __title__, __version__

py3 = sys.version_info[0] == 3
if py3:
    unicode = str


def get_parser():
    parser = ArgumentParser(description='convert chinese to pinyin.')
    parser.add_argument('-V', '--version', action='version',
                        version='{0} {1}'.format(__title__, __version__))
    # 要执行的函数名称
    parser.add_argument('--func', help='function name (default: "pinyin")',
                        choices=['pinyin', 'slug'],
                        default='pinyin')
    # 拼音风格
    parser.add_argument('--style', help='pinyin style (default: "TONE")',
                        choices=['NORMAL', 'TONE', 'TONE2', 'INITIALS',
                                 'FIRST_LETTER', 'FINALS', 'FINALS_TONE',
                                 'FINALS_TONE2'], default='TONE')
    parser.add_argument('--separator', help='slug separator (default: "-")',
                        default='-')
    # 输出多音字
    parser.add_argument('--heteronym', help='enable heteronym',
                        action='store_true')
    # 要查询的汉字
    parser.add_argument('hans', help='chinese string')
    return parser


def main():
    import logging
    # 禁用除 CRITICAL 外的日志消息
    logging.disable(logging.CRITICAL)

    # 获取命令行选项和参数
    parser = get_parser()
    options = parser.parse_args()
    if py3:
        hans = options.hans
    else:
        hans = options.hans.decode(sys.stdin.encoding)
    func = getattr(pypinyin, options.func)
    style = getattr(pypinyin, options.style)
    heteronym = options.heteronym
    separator = options.separator

    func_kwargs = {
        'pinyin': {'heteronym': heteronym},
        'slug': {'heteronym': heteronym, 'separator': separator},
    }
    if py3:
        kwargs = func_kwargs[func.__name__]
    else:
        kwargs = func_kwargs[func.func_name]
    result = func(hans, style=style, **kwargs)

    if result and isinstance(result, (list, tuple)):
        if isinstance(result[0], (list, tuple)):
            print(' '.join([','.join(s) for s in result]))
        else:
            print(result)
    else:
        print(result)

if __name__ == '__main__':
    main()
{'fraction_non_alphanumeric': 0.1380572905449063, 'fraction_numerical': 0.005169071720870127, 'mean_word_length': 2.4121969140337987, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6042772', 'n_tokens_mistral': 1515, 'n_tokens_neox': 1510, 'n_words': 312}