<commit_before># -*- coding: utf-8 -*-
"""
    pygments.formatters
    ~~~~~~~~~~~~~~~~~~

    Pygments formatters.

    :copyright: 2006 by <NAME>, <NAME>.
    :license: GNU LGPL, see LICENSE for more details.
"""
import os.path
from pygments.formatters.html import HtmlFormatter
from pygments.formatters.terminal import TerminalFormatter
from pygments.formatters.latex import LatexFormatter
from pygments.formatters.bbcode import BBCodeFormatter
from pygments.formatters.other import NullFormatter, RawTokenFormatter


def _doc_desc(obj):
    res = ''
    for line in obj.__doc__.strip().splitlines():
        if line.strip(): res += line.strip() + " "
        else: break<commit_msg>[svn] Avoid failing if no docstring given for a formatter class. Fixes #121.<commit_after># -*- coding: utf-8 -*-
"""
    pygments.formatters
    ~~~~~~~~~~~~~~~~~~

    Pygments formatters.

    :copyright: 2006 by <NAME>, <NAME>.
    :license: GNU LGPL, see LICENSE for more details.
"""
import os.path
from pygments.formatters.html import HtmlFormatter
from pygments.formatters.terminal import TerminalFormatter
from pygments.formatters.latex import LatexFormatter
from pygments.formatters.bbcode import BBCodeFormatter
from pygments.formatters.other import NullFormatter, RawTokenFormatter


def _doc_desc(obj):
    if not obj.__doc__:
        return ''
    res = ''
    for line in obj.__doc__.strip().splitlines():
        if line.strip(): res += line.strip() + " "
        else: break
{'fraction_non_alphanumeric': 0.12660798916723087, 'fraction_numerical': 0.008801624915368992, 'mean_word_length': 3.7831715210355985, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4646006', 'n_tokens_mistral': 450, 'n_tokens_neox': 414, 'n_words': 125}