<commit_before>import py
import sys
from inspect import CO_VARARGS, CO_VARKEYWORDS

builtin_repr = repr

reprlib = py.builtin._tryimport('repr', 'reprlib')

if sys.version_info[0] >= 3:
    from traceback import format_exception_only
else:
    from py._code._py2traceback import format_exception_only

class Code(object):
    """ wrapper around Python code objects """
    def __init__(self, rawcode):
        if not hasattr(rawcode, "co_filename"):
            rawcode = py.code.getrawcode(rawcode)
        try:
            self.filename = rawcode.co_filename
            self.firstlineno = rawcode.co_firstlineno - 1
            self.name = rawcode.co_name
        except AttributeError:
            raise TypeError("not a code object: %r" %(rawcode,))
        self.raw = rawcode

    def __eq__(self, other):
        return self.raw == other.raw

    def __ne__(self, other):
        return not self == other

    @property
    def path(self):
        """ return a path object pointing to source code (note that it
        might not point to an actually existing file). """
        p = py.path.local(self.raw.co_filename)
        # maybe don't try this checking
        if not p.check():
            # XXX maybe try harder like the weird logic
            # in the standard lib [linecache.updatecache] does?
            p = self.raw.co_filename
        return p

    @property
    def fullsource(self):
        """ return a py.code.Source object for the full source file of the code
        """
        from py._code import source
        full, _ = source.findsource(self.raw)
        return full

    def source(self):
        """ return a py.code.Source object for the code object's source only
        """
        # return source only for that part of code
        return py.code.Source(self.raw)

    def getargs(self, var=False):
        """ return a tuple with the argument names for the code object

            if 'var' is set True also return the names of the variable and
            keyword arguments when present
        """
        # handfull shortcut for getting args
        raw = self.raw
        argcount = raw.co_argcount
        if var:
            argcount += raw.co_flags & CO_VARARGS
            argcount += raw.co_flags & CO_VARKEYWORDS
        return raw.co_varnames[:argcount]

class Frame(object):
    """Wrapper around a Python frame holding f_locals and f_globals
    in which expressions can be evaluated."""

    def __init__(self, frame):
        self.lineno = frame.f_lineno - 1
        self.f_globals = frame.f_globals
        self.f_locals = frame.f_locals
        self.raw = frame
        self.code = py.code.Code(frame.f_code)

    @property
    def statement(self):
        """ statement this frame is at """
        if self.code.fullsource is None:
            return py.code.Source("")
        return self.code.fullsource.getstatement(self.lineno)

    def eval(self, code, **vars):
        """ evaluate 'code' in the frame

            'vars' are optional additional local variables

            returns the result of the evaluation
        """
        f_locals = self.f_locals.copy()
        f_locals.update(vars)
        return eval(code, self.f_globals, f_locals)

    def exec_(self, code, **vars):
        """ exec 'code' in the frame

            'vars' are optiona; additional local variables
        """
        f_locals = self.f_locals.copy()
        f_locals.update(vars)
        py.builtin.exec_(code, self.f_globals, f_locals )

    def repr(self, object):
        """ return a 'safe' (non-recursive, one-line) string repr for 'object'
        """
        return py.io.saferepr(object)

    def is_true(self, object):
        return object

    def getargs(self, var=False):
        """ return a list of tuples (name, value) for all arguments

            if 'var' is set True also include the variable and keyword
            arguments when present
        """
        retval = []
        for arg in self.code.getargs(var):
            try:
                retval.append((arg, self.f_locals[arg]))
            except KeyError:
                pass     # this can occur when using Psyco
        return retval

class TracebackEntry(object):
    """ a single entry in a traceback """

    exprinfo = None

    def __init__(self, rawentry):
        self._rawentry = rawentry
        self.lineno = rawentry.tb_lineno - 1

    @property
    def frame(self):
        return py.code.Frame(self._rawentry.tb_frame)

    @property
    def relline(self):
        return self.lineno - self.frame.code.firstlineno

    def __repr__(self):
        return "<TracebackEntry %s:%d>" %(self.frame.code.path, self.lineno+1)

    @property
    def statement(self):
        """ py.code.Source object for the current statement """
        source = self.frame.code.fullsource
        return source.getstatement(self.lineno)

    @property
    def path(self):
        """ path to the source code """
        return self.frame.code.path

    def getlocals(self):
        return self.frame.f_locals
    locals = property(getlocals, None, None, "locals of underlaying frame")

    def reinterpret(self):
        """Reinterpret the failing statement and returns a detailed information
           about what operations are performed."""
        if self.exprinfo is None:
            source = str(self.statement).strip()
            x = py.code._reinterpret(source, self.frame, should_fail=True)
            if not isinstance(x, str):
                raise TypeError("interpret returned non-string %r" % (x,))
            self.exprinfo = x
        return self.exprinfo

    def getfirstlinesource(self):
        # on Jython this firstlineno can be -1 apparently
        return max(self.frame.code.firstlineno, 0)

    def getsource(self, astcache=None):
        """ return failing source code. """
        # we use the passed in astcache to not reparse asttrees
        # within exception info printing
        from py._code.source import getstatementrange_ast
        source = self.frame.code.fullsource
        if source is None:
            return None
        key = astnode = None
        if astcache is not None:
            key = self.frame.code.path
            if key is not None:
                astnode = astcache.get(key, None)
        start = self.getfirstlinesource()
        try:
            astnode, _, end = getstatementrange_ast(self.lineno, source,
                                                    astnode=astnode)
        except SyntaxError:
            end = self.lineno + 1
        else:
            if key is not None:
                astcache[key] = astnode
        return source[start:end]

    source = property(getsource)

    def ishidden(self):
        """ return True if the current frame has a var __tracebackhide__
            resolving to True

            mostly for internal use
        """
        try:
            return self.frame.f_locals['__tracebackhide__']
        except KeyError:
            try:
                return self.frame.f_globals['__tracebackhide__']
            except KeyError:
                return False

    def __str__(self):
        try:
            fn = str(self.path)
        except py.error.Error:
            fn = '???'
        name = self.frame.code.name
        try:
            line = str(self.statement).lstrip()
        except KeyboardInterrupt:
            raise
        except:
            line = "???"
        return "  File %r:%d in %s\n  %s\n" %(fn, self.lineno+1, name, line)

    def name(self):
        return self.frame.code.raw.co_name
    name = property(name, None, None, "co_name of underlaying code")

class Traceback(list):
    """ Traceback objects encapsulate and offer higher level
        access to Traceback entries.
    """
    Entry = TracebackEntry
    def __init__(self, tb):
        """ initialize from given python traceback object. """
        if hasattr(tb, 'tb_next'):
            def f(cur):
                while cur is not None:
                    yield self.Entry(cur)
                    cur = cur.tb_next
            list.__init__(self, f(tb))
        else:
            list.__init__(self, tb)

    def cut(self, path=None, lineno=None, firstlineno=None, excludepath=None):
        """ return a Traceback instance wrapping part of this Traceback

            by provding any combination of path, lineno and firstlineno, the
            first frame to start the to-be-returned traceback is determined

            this allows cutting the first part of a Traceback instance e.g.
            for formatting reasons (removing some uninteresting bits that deal
            with handling of the exception/traceback)
        """
        for x in self:
            code = x.frame.code
            codepath = code.path
            if ((path is None or codepath == path) and
                (excludepath is None or not hasattr(codepath, 'relto') or
                 not codepath.relto(excludepath)) and
                (lineno is None or x.lineno == lineno) and
                (firstlineno is None or x.frame.code.firstlineno == firstlineno)):
                return Traceback(x._rawentry)
        return self

    def __getitem__(self, key):
        val = super(Traceback, self).__getitem__(key)
        if isinstance(key, type(slice(0))):
            val = self.__class__(val)
        return val

    def filter(self, fn=lambda x: not x.ishidden()):
        """ return a Traceback instance with certain items removed

            fn is a function that gets a single argument, a TracebackItem
            instance, and should return True when the item should be added
            to the Traceback, False when not

            by default this removes all the TracebackItems which are hidden
            (see ishidden() above)
        """
        return Traceback(filter(fn, self))

    def getcrashentry(self):
        """ return last non-hidden traceback entry that lead
        to the exception of a traceback.
        """
        for i in range(-1, -len(self)-1, -1):
            entry = self[i]
            if not entry.ishidden():
                return entry
        return self[-1]

    def recursionindex(self):
        """ return the index of the frame/TracebackItem where recursion
            originates if appropriate, None if no recursion occurred
        """
        cache = {}
        for i, entry in enumerate(self):
            # id for the code.raw is needed to work around
            # the strange metaprogramming in the decorator lib from pypi
            # which generates code objects that have hash/value equality
            #XXX needs a test
            key = entry.frame.code.path, id(entry.frame.code.raw), entry.lineno
            #print "checking for recursion at", key
            l = cache.setdefault(key, [])
            if l:
                f = entry.frame
                loc = f.f_locals
                for otherloc in l:
                    if f.is_true(f.eval(co_equal,
                        __recursioncache_locals_1=loc,
                        __recursioncache_locals_2=otherloc)):
                        return i
            l.append(entry.frame.f_locals)
        return None

co_equal = compile('__recursioncache_locals_1 == __recursioncache_locals_2',
                   '?', 'eval')

class ExceptionInfo(object):
    """ wraps sys.exc_info() objects and offers
        help for navigating the traceback.
    """
    _striptext = ''
    def __init__(self, tup=None, exprinfo=None):
        # NB. all attributes are private!  Subclasses or other
        #     ExceptionInfo-like classes may have different attributes.
        if tup is None:
            tup = sys.exc_info()
            if exprinfo is None and isinstance(tup[1], AssertionError):
                exprinfo = getattr(tup[1], 'msg', None)
                if exprinfo is None:
                    exprinfo = str(tup[1])
                if exprinfo and exprinfo.startswith('assert '):
                    self._striptext = 'AssertionError: '
        self._excinfo = tup
        #: the exception class
        self.type = tup[0]
        #: the exception instance<commit_msg>remove comment about ExceptionInfo attributes being private.<commit_after>import py
import sys
from inspect import CO_VARARGS, CO_VARKEYWORDS

builtin_repr = repr

reprlib = py.builtin._tryimport('repr', 'reprlib')

if sys.version_info[0] >= 3:
    from traceback import format_exception_only
else:
    from py._code._py2traceback import format_exception_only

class Code(object):
    """ wrapper around Python code objects """
    def __init__(self, rawcode):
        if not hasattr(rawcode, "co_filename"):
            rawcode = py.code.getrawcode(rawcode)
        try:
            self.filename = rawcode.co_filename
            self.firstlineno = rawcode.co_firstlineno - 1
            self.name = rawcode.co_name
        except AttributeError:
            raise TypeError("not a code object: %r" %(rawcode,))
        self.raw = rawcode

    def __eq__(self, other):
        return self.raw == other.raw

    def __ne__(self, other):
        return not self == other

    @property
    def path(self):
        """ return a path object pointing to source code (note that it
        might not point to an actually existing file). """
        p = py.path.local(self.raw.co_filename)
        # maybe don't try this checking
        if not p.check():
            # XXX maybe try harder like the weird logic
            # in the standard lib [linecache.updatecache] does?
            p = self.raw.co_filename
        return p

    @property
    def fullsource(self):
        """ return a py.code.Source object for the full source file of the code
        """
        from py._code import source
        full, _ = source.findsource(self.raw)
        return full

    def source(self):
        """ return a py.code.Source object for the code object's source only
        """
        # return source only for that part of code
        return py.code.Source(self.raw)

    def getargs(self, var=False):
        """ return a tuple with the argument names for the code object

            if 'var' is set True also return the names of the variable and
            keyword arguments when present
        """
        # handfull shortcut for getting args
        raw = self.raw
        argcount = raw.co_argcount
        if var:
            argcount += raw.co_flags & CO_VARARGS
            argcount += raw.co_flags & CO_VARKEYWORDS
        return raw.co_varnames[:argcount]

class Frame(object):
    """Wrapper around a Python frame holding f_locals and f_globals
    in which expressions can be evaluated."""

    def __init__(self, frame):
        self.lineno = frame.f_lineno - 1
        self.f_globals = frame.f_globals
        self.f_locals = frame.f_locals
        self.raw = frame
        self.code = py.code.Code(frame.f_code)

    @property
    def statement(self):
        """ statement this frame is at """
        if self.code.fullsource is None:
            return py.code.Source("")
        return self.code.fullsource.getstatement(self.lineno)

    def eval(self, code, **vars):
        """ evaluate 'code' in the frame

            'vars' are optional additional local variables

            returns the result of the evaluation
        """
        f_locals = self.f_locals.copy()
        f_locals.update(vars)
        return eval(code, self.f_globals, f_locals)

    def exec_(self, code, **vars):
        """ exec 'code' in the frame

            'vars' are optiona; additional local variables
        """
        f_locals = self.f_locals.copy()
        f_locals.update(vars)
        py.builtin.exec_(code, self.f_globals, f_locals )

    def repr(self, object):
        """ return a 'safe' (non-recursive, one-line) string repr for 'object'
        """
        return py.io.saferepr(object)

    def is_true(self, object):
        return object

    def getargs(self, var=False):
        """ return a list of tuples (name, value) for all arguments

            if 'var' is set True also include the variable and keyword
            arguments when present
        """
        retval = []
        for arg in self.code.getargs(var):
            try:
                retval.append((arg, self.f_locals[arg]))
            except KeyError:
                pass     # this can occur when using Psyco
        return retval

class TracebackEntry(object):
    """ a single entry in a traceback """

    exprinfo = None

    def __init__(self, rawentry):
        self._rawentry = rawentry
        self.lineno = rawentry.tb_lineno - 1

    @property
    def frame(self):
        return py.code.Frame(self._rawentry.tb_frame)

    @property
    def relline(self):
        return self.lineno - self.frame.code.firstlineno

    def __repr__(self):
        return "<TracebackEntry %s:%d>" %(self.frame.code.path, self.lineno+1)

    @property
    def statement(self):
        """ py.code.Source object for the current statement """
        source = self.frame.code.fullsource
        return source.getstatement(self.lineno)

    @property
    def path(self):
        """ path to the source code """
        return self.frame.code.path

    def getlocals(self):
        return self.frame.f_locals
    locals = property(getlocals, None, None, "locals of underlaying frame")

    def reinterpret(self):
        """Reinterpret the failing statement and returns a detailed information
           about what operations are performed."""
        if self.exprinfo is None:
            source = str(self.statement).strip()
            x = py.code._reinterpret(source, self.frame, should_fail=True)
            if not isinstance(x, str):
                raise TypeError("interpret returned non-string %r" % (x,))
            self.exprinfo = x
        return self.exprinfo

    def getfirstlinesource(self):
        # on Jython this firstlineno can be -1 apparently
        return max(self.frame.code.firstlineno, 0)

    def getsource(self, astcache=None):
        """ return failing source code. """
        # we use the passed in astcache to not reparse asttrees
        # within exception info printing
        from py._code.source import getstatementrange_ast
        source = self.frame.code.fullsource
        if source is None:
            return None
        key = astnode = None
        if astcache is not None:
            key = self.frame.code.path
            if key is not None:
                astnode = astcache.get(key, None)
        start = self.getfirstlinesource()
        try:
            astnode, _, end = getstatementrange_ast(self.lineno, source,
                                                    astnode=astnode)
        except SyntaxError:
            end = self.lineno + 1
        else:
            if key is not None:
                astcache[key] = astnode
        return source[start:end]

    source = property(getsource)

    def ishidden(self):
        """ return True if the current frame has a var __tracebackhide__
            resolving to True

            mostly for internal use
        """
        try:
            return self.frame.f_locals['__tracebackhide__']
        except KeyError:
            try:
                return self.frame.f_globals['__tracebackhide__']
            except KeyError:
                return False

    def __str__(self):
        try:
            fn = str(self.path)
        except py.error.Error:
            fn = '???'
        name = self.frame.code.name
        try:
            line = str(self.statement).lstrip()
        except KeyboardInterrupt:
            raise
        except:
            line = "???"
        return "  File %r:%d in %s\n  %s\n" %(fn, self.lineno+1, name, line)

    def name(self):
        return self.frame.code.raw.co_name
    name = property(name, None, None, "co_name of underlaying code")

class Traceback(list):
    """ Traceback objects encapsulate and offer higher level
        access to Traceback entries.
    """
    Entry = TracebackEntry
    def __init__(self, tb):
        """ initialize from given python traceback object. """
        if hasattr(tb, 'tb_next'):
            def f(cur):
                while cur is not None:
                    yield self.Entry(cur)
                    cur = cur.tb_next
            list.__init__(self, f(tb))
        else:
            list.__init__(self, tb)

    def cut(self, path=None, lineno=None, firstlineno=None, excludepath=None):
        """ return a Traceback instance wrapping part of this Traceback

            by provding any combination of path, lineno and firstlineno, the
            first frame to start the to-be-returned traceback is determined

            this allows cutting the first part of a Traceback instance e.g.
            for formatting reasons (removing some uninteresting bits that deal
            with handling of the exception/traceback)
        """
        for x in self:
            code = x.frame.code
            codepath = code.path
            if ((path is None or codepath == path) and
                (excludepath is None or not hasattr(codepath, 'relto') or
                 not codepath.relto(excludepath)) and
                (lineno is None or x.lineno == lineno) and
                (firstlineno is None or x.frame.code.firstlineno == firstlineno)):
                return Traceback(x._rawentry)
        return self

    def __getitem__(self, key):
        val = super(Traceback, self).__getitem__(key)
        if isinstance(key, type(slice(0))):
            val = self.__class__(val)
        return val

    def filter(self, fn=lambda x: not x.ishidden()):
        """ return a Traceback instance with certain items removed

            fn is a function that gets a single argument, a TracebackItem
            instance, and should return True when the item should be added
            to the Traceback, False when not

            by default this removes all the TracebackItems which are hidden
            (see ishidden() above)
        """
        return Traceback(filter(fn, self))

    def getcrashentry(self):
        """ return last non-hidden traceback entry that lead
        to the exception of a traceback.
        """
        for i in range(-1, -len(self)-1, -1):
            entry = self[i]
            if not entry.ishidden():
                return entry
        return self[-1]

    def recursionindex(self):
        """ return the index of the frame/TracebackItem where recursion
            originates if appropriate, None if no recursion occurred
        """
        cache = {}
        for i, entry in enumerate(self):
            # id for the code.raw is needed to work around
            # the strange metaprogramming in the decorator lib from pypi
            # which generates code objects that have hash/value equality
            #XXX needs a test
            key = entry.frame.code.path, id(entry.frame.code.raw), entry.lineno
            #print "checking for recursion at", key
            l = cache.setdefault(key, [])
            if l:
                f = entry.frame
                loc = f.f_locals
                for otherloc in l:
                    if f.is_true(f.eval(co_equal,
                        __recursioncache_locals_1=loc,
                        __recursioncache_locals_2=otherloc)):
                        return i
            l.append(entry.frame.f_locals)
        return None

co_equal = compile('__recursioncache_locals_1 == __recursioncache_locals_2',
                   '?', 'eval')

class ExceptionInfo(object):
    """ wraps sys.exc_info() objects and offers
        help for navigating the traceback.
    """
    _striptext = ''
    def __init__(self, tup=None, exprinfo=None):
        if tup is None:
            tup = sys.exc_info()
            if exprinfo is None and isinstance(tup[1], AssertionError):
                exprinfo = getattr(tup[1], 'msg', None)
                if exprinfo is None:
                    exprinfo = str(tup[1])
                if exprinfo and exprinfo.startswith('assert '):
                    self._striptext = 'AssertionError: '
        self._excinfo = tup
        #: the exception class
        self.type = tup[0]
        #: the exception instance
{'fraction_non_alphanumeric': 0.08623404607989392, 'fraction_numerical': 0.001989060169070114, 'mean_word_length': 1.9075903614457832, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '61171', 'n_tokens_mistral': 6696, 'n_tokens_neox': 6504, 'n_words': 2105}