<commit_before># -*- coding: utf-8 -*-
"""
    pygments.formatters.other
    ~~~~~~~~~~~~~~~~~~~~~~~~~

    Other formatters: NullFormatter, RawTokenFormatter.

    :copyright: 2006 by <NAME>, <NAME>.
    :license: BSD, see LICENSE for more details.
"""

from pygments.formatter import Formatter


__all__ = ['NullFormatter', 'RawTokenFormatter']


class NullFormatter(Formatter):
    """
    Output the text unchanged without any formatting.
    """
    def format(self, tokensource, outfile):
        for ttype, value in tokensource:
            outfile.write(value.encode(self.encoding))


class RawTokenFormatter(Formatter):
    """
    Output a raw token representation for storing token streams.

    The format is ``tokentype<TAB>repr(tokenstring)``

    Additional options accepted:

    ``compress``
        If set to "gz" or "bz2", compress the token stream with
        the given compression algorithm (default: '').
    """

    def __init__(self, **options):
        Formatter.__init__(self, **options)
        self.compress = options.get('compress', '')

    def format(self, tokensource, outfile):
        if self.compress == 'gz':
            import gzip
            outfile = gzip.GzipFile('', 'wb', 9, outfile)
            write = outfile.write
            flush = outfile.flush
        elif self.compress == 'bz2':
            import bz2
            compressor = bz2.BZ2Compressor(9)
            def write(text):
                outfile.write(compressor.compress(text))
            def flush():
                outfile.write(compressor.flush())
                outfile.flush()
        else:
            write = outfile.write
            flush = outfile.flush

        lasttype = None
        lastval = u''
        for ttype, value in tokensource:
            value = repr(value)
            if ttype is lasttype:
                lastval += value
            else:
                if lasttype:
                    write("%s\t%r\n" % (lasttype, lastval))
                lastval = value
                lasttype = ttype
        write("%s\t%r\n" % (lasttype, lastval))
        flush()<commit_msg>[svn] Fix double repr.<commit_after># -*- coding: utf-8 -*-
"""
    pygments.formatters.other
    ~~~~~~~~~~~~~~~~~~~~~~~~~

    Other formatters: NullFormatter, RawTokenFormatter.

    :copyright: 2006 by <NAME>, <NAME>.
    :license: BSD, see LICENSE for more details.
"""

from pygments.formatter import Formatter


__all__ = ['NullFormatter', 'RawTokenFormatter']


class NullFormatter(Formatter):
    """
    Output the text unchanged without any formatting.
    """
    def format(self, tokensource, outfile):
        for ttype, value in tokensource:
            outfile.write(value.encode(self.encoding))


class RawTokenFormatter(Formatter):
    """
    Output a raw token representation for storing token streams.

    The format is ``tokentype<TAB>repr(tokenstring)``

    Additional options accepted:

    ``compress``
        If set to "gz" or "bz2", compress the token stream with
        the given compression algorithm (default: '').
    """

    def __init__(self, **options):
        Formatter.__init__(self, **options)
        self.compress = options.get('compress', '')

    def format(self, tokensource, outfile):
        if self.compress == 'gz':
            import gzip
            outfile = gzip.GzipFile('', 'wb', 9, outfile)
            write = outfile.write
            flush = outfile.flush
        elif self.compress == 'bz2':
            import bz2
            compressor = bz2.BZ2Compressor(9)
            def write(text):
                outfile.write(compressor.compress(text))
            def flush():
                outfile.write(compressor.flush())
                outfile.flush()
        else:
            write = outfile.write
            flush = outfile.flush

        lasttype = None
        lastval = u''
        for ttype, value in tokensource:
            value = repr(value)
            if ttype is lasttype:
                lastval += value
            else:
                if lasttype:
                    write("%s\t%s\n" % (lasttype, lastval))
                lastval = value
                lasttype = ttype
        write("%s\t%s\n" % (lasttype, lastval))
        flush()
{'fraction_non_alphanumeric': 0.12024568863690055, 'fraction_numerical': 0.005669737774627924, 'mean_word_length': 1.9795918367346939, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4412801', 'n_tokens_mistral': 1202, 'n_tokens_neox': 1125, 'n_words': 314}