<commit_before>#!/usr/bin/python
# -*- coding: utf-8 -*-
#
# pip install CppHeaderParser ply
#

#
# how to use api_reference.py
#
# 1. run `./tools/api_reference.py`
#
# 2. check documents
#     - documentation/PluginXX/api-reference-cpp.md
#     - documentation/PluginXX/api-reference-lua.md
#     - documentation/PluginXX/api-reference-js.md
#
# how to config
# 1. manual binding function
#   - edit `plugins/pluginx/lua/config/binding.ini`
#   - add section `manual_binding_function = PluginXX::[setListener,otherFunction]`
#
# 2. rename callback function name (optional)
#   - edit `plugins/pluginx/lua/config/binding.ini`
#   - add section `rename_script_callback = PluginXX::[onCallbackLongFunction=onCallbackFunction]`
#     before `[headers]`
#

import ConfigParser
import os
import sys
import CppHeaderParser

from argparse import ArgumentParser
from sets import Set


CURR_DIR = os.path.split(os.path.realpath(__file__))[0]
OUT_DIR = os.path.realpath(CURR_DIR + "./..")

PLUGIN_PREFIX = "Plugin"


def get_doxygen(comment):
    if '\r\n' in comment:
        cs = comment.split('\r\n')
    else:
        cs = comment.split('\n')

    ret = ""
    for line in cs:
        if line.startswith("/*") or line.endswith("*/"):
            continue
        elif line == "*":  # brief end with empty line
            break
        else:
            if line.startswith("* "):
                line = line[2:]
            ret += line + '\n'

    ret = ret.replace("@brief ", "")
    return ret


def format_function(function_string):
    """
    static void some_function(int param1, int param2);
    =>
    static void some_function(int param1,
                              int param2)
    """
    ret = function_string
    if len(ret) <= 80:
        return ret

    ret = ret.split(',')
    index = ret[0].rfind('(')
    for x in xrange(1, len(ret)):
        ret[x] = " %s%s" % (index * " ", ret[x])

    ret = ',\n'.join(ret)
    return ret


class APIRefGenerator(object):<commit_msg>Generate more comment<commit_after>#!/usr/bin/python
# -*- coding: utf-8 -*-
#
# pip install CppHeaderParser ply
#

#
# how to use api_reference.py
#
# 1. run `./tools/api_reference.py`
#
# 2. check documents
#     - documentation/PluginXX/api-reference-cpp.md
#     - documentation/PluginXX/api-reference-lua.md
#     - documentation/PluginXX/api-reference-js.md
#
# how to config
# 1. manual binding function
#   - edit `plugins/pluginx/lua/config/binding.ini`
#   - add section `manual_binding_function = PluginXX::[setListener,otherFunction]`
#
# 2. rename callback function name (optional)
#   - edit `plugins/pluginx/lua/config/binding.ini`
#   - add section `rename_script_callback = PluginXX::[onCallbackLongFunction=onCallbackFunction]`
#     before `[headers]`
#

import ConfigParser
import os
import sys
import CppHeaderParser

from argparse import ArgumentParser
from sets import Set


CURR_DIR = os.path.split(os.path.realpath(__file__))[0]
OUT_DIR = os.path.realpath(CURR_DIR + "./..")

PLUGIN_PREFIX = "Plugin"


def get_doxygen(comment):
    if '\r\n' in comment:
        cs = comment.split('\r\n')
    else:
        cs = comment.split('\n')

    ret = ""
    has_more_comment = False
    for line in cs:
        if line.startswith("/*") or line.endswith("*/") or line.startswith("* @param"):
            continue
        elif line == "*":  # brief end with empty line
            if not has_more_comment:
                ret += '\n<pre>\n'
            has_more_comment = True
        else:
            if line.startswith("* "):
                line = line[2:]
            ret += line + '\n'

    if has_more_comment:
        ret += '</pre>\n'

    ret = ret.replace("@brief ", "").replace('*', '>').replace('\n<pre>\n</pre>\n', '')
    return ret


def format_function(function_string):
    """
    static void some_function(int param1, int param2);
    =>
    static void some_function(int param1,
                              int param2)
    """
    ret = function_string
    if len(ret) <= 80:
        return ret

    ret = ret.split(',')
    index = ret[0].rfind('(')
    for x in xrange(1, len(ret)):
        ret[x] = " %s%s" % (index * " ", ret[x])

    ret = ',\n'.join(ret)
    return ret


class APIRefGenerator(object):
{'fraction_non_alphanumeric': 0.14764942121426883, 'fraction_numerical': 0.007087172218284904, 'mean_word_length': 2.54606365159129, 'pattern_counts': {'":': 2, '<': 9, '<?xml version=': 0, '>': 10, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5412203', 'n_tokens_mistral': 1429, 'n_tokens_neox': 1389, 'n_words': 318}