<commit_before>#!/usr/bin/python
# -*- coding: utf-8 -*-
"""
    Lexing error finder
    ~~~~~~~~~~~~~~~~~~~

    For the source files given on the command line, display
    the text where Error tokens are being generated, along
    with some context.

    :copyright: Copyright 2006-2014 by the Pygments team, see AUTHORS.
    :license: BSD, see LICENSE for details.
"""

from __future__ import print_function

import os
import sys

# always prefer Pygments from source if exists
srcpath = os.path.join(os.path.dirname(__file__), '..')
if os.path.isdir(os.path.join(srcpath, 'pygments')):
    sys.path.insert(0, srcpath)


from pygments.lexer import RegexLexer, ProfilingRegexLexer, ProfilingRegexLexerMeta
from pygments.lexers import get_lexer_by_name, find_lexer_class_for_filename
from pygments.token import Error, Text, _TokenType
from pygments.cmdline import _parse_options


class DebuggingRegexLexer(RegexLexer):
    """Make the state stack, position and current match instance attributes."""

    def get_tokens_unprocessed(self, text, stack=('root',)):
        """
        Split ``text`` into (tokentype, text) pairs.

        ``stack`` is the inital stack (default: ``['root']``)
        """
        self.pos = 0
        tokendefs = self._tokens
        self.statestack = list(stack)
        statetokens = tokendefs[self.statestack[-1]]
        while 1:
            for rexmatch, action, new_state in statetokens:
                self.m = m = rexmatch(text, self.pos)
                if m:
                    if action is not None:
                        if type(action) is _TokenType:
                            yield self.pos, action, m.group()
                        else:
                            for item in action(self, m):
                                yield item
                    self.pos = m.end()
                    if new_state is not None:
                        # state transition
                        if isinstance(new_state, tuple):
                            for state in new_state:
                                if state == '#pop':
                                    self.statestack.pop()
                                elif state == '#push':
                                    self.statestack.append(self.statestack[-1])
                                else:
                                    self.statestack.append(state)
                        elif isinstance(new_state, int):
                            # pop
                            del self.statestack[new_state:]
                        elif new_state == '#push':
                            self.statestack.append(self.statestack[-1])
                        else:
                            assert False, 'wrong state def: %r' % new_state
                        statetokens = tokendefs[self.statestack[-1]]
                    break
            else:
                try:
                    if text[self.pos] == '\n':
                        # at EOL, reset state to 'root'
                        self.pos += 1
                        self.statestack = ['root']
                        statetokens = tokendefs['root']
                        yield self.pos, Text, u'\n'
                        continue
                    yield self.pos, Error, text[self.pos]
                    self.pos += 1
                except IndexError:
                    break


def main(fn, lexer=None, options={}):
    if lexer is not None:
        lxcls = get_lexer_by_name(lexer).__class__
    else:
        lxcls = find_lexer_class_for_filename(os.path.basename(fn))
        if lxcls is None:
            try:
                name, rest = fn.split('_', 1)
                lxcls = find_lexer_class(name)
            except ValueError:
                raise AssertionError('no lexer found for file %r' % fn)
    debug_lexer = False
    if profile:
        # does not work for e.g. ExtendedRegexLexers
        if lxcls.__bases__ == (RegexLexer,):
            # yes we can!  (change the metaclass)
            lxcls.__class__ = ProfilingRegexLexerMeta
            lxcls.__bases__ = (ProfilingRegexLexer,)
            lxcls._prof_sort_index = profsort
    else:
        if lxcls.__bases__ == (RegexLexer,):
            lxcls.__bases__ = (DebuggingRegexLexer,)
            debug_lexer = True
        elif lxcls.__bases__ == (DebuggingRegexLexer,):
            # already debugged before
            debug_lexer = True
        else:
            # HACK: ExtendedRegexLexer subclasses will only partially work here.
            print(lxcls.__bases__)
            lxcls.__bases__ = (DebuggingRegexLexer,)
            debug_lexer = True

    lx = lxcls(**options)
    lno = 1
    text = open(fn, 'rb').read().decode('utf-8')
    text = text.strip('\n') + '\n'
    tokens = []
    states = []

    def show_token(tok, state):
        reprs = list(map(repr, tok))
        print('   ' + reprs[1] + ' ' + ' ' * (29-len(reprs[1])) + reprs[0], end=' ')
        if debug_lexer:
            print(' ' + ' ' * (29-len(reprs[0])) + repr(state), end=' ')
        print()

    for type, val in lx.get_tokens(text):
        lno += val.count('\n')
        if type == Error:
            print('Error parsing', fn, 'on line', lno)
            print('Previous tokens' + (debug_lexer and ' and states' or '') + ':')
            if showall:
                for tok, state in map(None, tokens, states):
                    show_token(tok, state)
            else:
                for i in range(max(len(tokens) - num, 0), len(tokens)):
                    if debug_lexer:
                        show_token(tokens[i], states[i])
                    else:
                        show_token(tokens[i], None)
            print('Error token:')
            l = len(repr(val))
            print('   ' + repr(val), end=' ')
            if debug_lexer and hasattr(lx, 'statestack'):<commit_msg>debug_lexer: remove debug print; allow reading from stdin with filename "-"<commit_after>#!/usr/bin/python
# -*- coding: utf-8 -*-
"""
    Lexing error finder
    ~~~~~~~~~~~~~~~~~~~

    For the source files given on the command line, display
    the text where Error tokens are being generated, along
    with some context.

    :copyright: Copyright 2006-2014 by the Pygments team, see AUTHORS.
    :license: BSD, see LICENSE for details.
"""

from __future__ import print_function

import os
import sys

# always prefer Pygments from source if exists
srcpath = os.path.join(os.path.dirname(__file__), '..')
if os.path.isdir(os.path.join(srcpath, 'pygments')):
    sys.path.insert(0, srcpath)


from pygments.lexer import RegexLexer, ProfilingRegexLexer, ProfilingRegexLexerMeta
from pygments.lexers import get_lexer_by_name, find_lexer_class_for_filename
from pygments.token import Error, Text, _TokenType
from pygments.cmdline import _parse_options


class DebuggingRegexLexer(RegexLexer):
    """Make the state stack, position and current match instance attributes."""

    def get_tokens_unprocessed(self, text, stack=('root',)):
        """
        Split ``text`` into (tokentype, text) pairs.

        ``stack`` is the inital stack (default: ``['root']``)
        """
        self.pos = 0
        tokendefs = self._tokens
        self.statestack = list(stack)
        statetokens = tokendefs[self.statestack[-1]]
        while 1:
            for rexmatch, action, new_state in statetokens:
                self.m = m = rexmatch(text, self.pos)
                if m:
                    if action is not None:
                        if type(action) is _TokenType:
                            yield self.pos, action, m.group()
                        else:
                            for item in action(self, m):
                                yield item
                    self.pos = m.end()
                    if new_state is not None:
                        # state transition
                        if isinstance(new_state, tuple):
                            for state in new_state:
                                if state == '#pop':
                                    self.statestack.pop()
                                elif state == '#push':
                                    self.statestack.append(self.statestack[-1])
                                else:
                                    self.statestack.append(state)
                        elif isinstance(new_state, int):
                            # pop
                            del self.statestack[new_state:]
                        elif new_state == '#push':
                            self.statestack.append(self.statestack[-1])
                        else:
                            assert False, 'wrong state def: %r' % new_state
                        statetokens = tokendefs[self.statestack[-1]]
                    break
            else:
                try:
                    if text[self.pos] == '\n':
                        # at EOL, reset state to 'root'
                        self.pos += 1
                        self.statestack = ['root']
                        statetokens = tokendefs['root']
                        yield self.pos, Text, u'\n'
                        continue
                    yield self.pos, Error, text[self.pos]
                    self.pos += 1
                except IndexError:
                    break


def main(fn, lexer=None, options={}):
    if lexer is not None:
        lxcls = get_lexer_by_name(lexer).__class__
    else:
        lxcls = find_lexer_class_for_filename(os.path.basename(fn))
        if lxcls is None:
            try:
                name, rest = fn.split('_', 1)
                lxcls = find_lexer_class(name)
            except ValueError:
                raise AssertionError('no lexer found for file %r' % fn)
    debug_lexer = False
    if profile:
        # does not work for e.g. ExtendedRegexLexers
        if lxcls.__bases__ == (RegexLexer,):
            # yes we can!  (change the metaclass)
            lxcls.__class__ = ProfilingRegexLexerMeta
            lxcls.__bases__ = (ProfilingRegexLexer,)
            lxcls._prof_sort_index = profsort
    else:
        if lxcls.__bases__ == (RegexLexer,):
            lxcls.__bases__ = (DebuggingRegexLexer,)
            debug_lexer = True
        elif lxcls.__bases__ == (DebuggingRegexLexer,):
            # already debugged before
            debug_lexer = True
        else:
            # HACK: ExtendedRegexLexer subclasses will only partially work here.
            lxcls.__bases__ = (DebuggingRegexLexer,)
            debug_lexer = True

    lx = lxcls(**options)
    lno = 1
    if fn == '-':
        text = sys.stdin.read()
    else:
        with open(fn, 'rb') as fp:
            text = fp.read().decode('utf-8')
    text = text.strip('\n') + '\n'
    tokens = []
    states = []

    def show_token(tok, state):
        reprs = list(map(repr, tok))
        print('   ' + reprs[1] + ' ' + ' ' * (29-len(reprs[1])) + reprs[0], end=' ')
        if debug_lexer:
            print(' ' + ' ' * (29-len(reprs[0])) + repr(state), end=' ')
        print()

    for type, val in lx.get_tokens(text):
        lno += val.count('\n')
        if type == Error:
            print('Error parsing', fn, 'on line', lno)
            print('Previous tokens' + (debug_lexer and ' and states' or '') + ':')
            if showall:
                for tok, state in map(None, tokens, states):
                    show_token(tok, state)
            else:
                for i in range(max(len(tokens) - num, 0), len(tokens)):
                    if debug_lexer:
                        show_token(tokens[i], states[i])
                    else:
                        show_token(tokens[i], None)
            print('Error token:')
            l = len(repr(val))
            print('   ' + repr(val), end=' ')
            if debug_lexer and hasattr(lx, 'statestack'):
{'fraction_non_alphanumeric': 0.10584558202399054, 'fraction_numerical': 0.00506842372022301, 'mean_word_length': 1.4659445948760674, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 8, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5073275', 'n_tokens_mistral': 3479, 'n_tokens_neox': 3221, 'n_words': 851}