<commit_before># -*- coding: utf-8 -*-
from __future__ import print_function, division, absolute_import, unicode_literals
import sys
import re
from contextlib import contextmanager
from .util import memoize
from . import buffering
from .exceptions import *  # @UnusedWildImport
from .ast import AST


class AbstractParserMixin(object):
    def _find_semantic_rule(self, name):
        result = getattr(self, name, None)
        if result is None or not isinstance(result, type(self._find_rule)):
            raise MissingSemanticFor(name)
        return result


class Parser(object):
    def __init__(self, text,
                        whitespace=None,
                        comments_re=None,
                        ignorecase=False,
                        verbose=False,
                        bufferClass=buffering.Buffer):
        self.text = text
        self.whitespace = set(whitespace if whitespace else '\t\v\n\r ')
        self.comments_re = comments_re
        self.ignorecase = ignorecase
        self.verbose = verbose
        self._bufferClass = bufferClass
        self._buffer = None
        self._ast_stack = []
        self._concrete_stack = []
        self._rule_stack = []
        self._cut_stack = [False]

    def parse(self, rule_name):
        try:
            self._buffer = self._bufferClass(self.text, self.whitespace)
            self._push_ast()
            self._concrete_stack.append([])
            self._call(rule_name, rule_name)
            return self.ast
        finally:
            del self._ast_stack[1:]

    @property
    def ast(self):
        return self._ast_stack[-1]

    def result(self):
        return self.ast

    def rulestack(self):
        stack = '.'.join(self._rule_stack)
        if len(stack) > 60:
            stack = '...' + stack[-60:]<commit_msg>Optimized trace when verbose is off. It's notable that this and previous optimizations shave only seconds (0.0x%) off the test suite for COBOL.<commit_after># -*- coding: utf-8 -*-
from __future__ import print_function, division, absolute_import, unicode_literals
import sys
import re
from contextlib import contextmanager
from .util import memoize
from . import buffering
from .exceptions import *  # @UnusedWildImport
from .ast import AST


class AbstractParserMixin(object):
    def _find_semantic_rule(self, name):
        result = getattr(self, name, None)
        if result is None or not isinstance(result, type(self._find_rule)):
            raise MissingSemanticFor(name)
        return result


class Parser(object):
    def __init__(self, text,
                        whitespace=None,
                        comments_re=None,
                        ignorecase=False,
                        verbose=False,
                        bufferClass=buffering.Buffer):
        self.text = text
        self.whitespace = set(whitespace if whitespace else '\t\v\n\r ')
        self.comments_re = comments_re
        self.ignorecase = ignorecase
        self.verbose = verbose
        self._bufferClass = bufferClass
        self._buffer = None
        self._ast_stack = []
        self._concrete_stack = []
        self._rule_stack = []
        self._cut_stack = [False]
        if not self.verbose:
            self.trace_event = lambda x: ()

    def parse(self, rule_name):
        try:
            self._buffer = self._bufferClass(self.text, self.whitespace)
            self._push_ast()
            self._concrete_stack.append([])
            self._call(rule_name, rule_name)
            return self.ast
        finally:
            del self._ast_stack[1:]

    @property
    def ast(self):
        return self._ast_stack[-1]

    def result(self):
        return self.ast

    def rulestack(self):
        stack = '.'.join(self._rule_stack)
        if len(stack) > 60:
            stack = '...' + stack[-60:]
{'fraction_non_alphanumeric': 0.09183407252804592, 'fraction_numerical': 0.004174276024002087, 'mean_word_length': 2.0236593059936907, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '757624', 'n_tokens_mistral': 1093, 'n_tokens_neox': 1051, 'n_words': 269}