<commit_before>import inspect
import os
import pprint
import struct
import warnings

try:
    import fcntl
except ImportError:
    fcntl = None
else:
    import termios

try:
    import pygments
except ImportError:
    pygments = None
else:
    from pygments.formatters.terminal import TerminalFormatter
    from .lexer import PythonXTracebackLexer

from .xtracebackframe import XTracebackFrame
from .xtracebackoptions import XTracebackOptions


class XTraceback(object):
    """
    An extended traceback formatter

    """

    #: The default width of one line
    DEFAULT_WIDTH = 80

    #: A dictionary keyed by type with values as the wrapper for string
    #: representation of the type
    REFORMAT = {
        dict: ("{", "}"),
        list: ("[", "]"),
        tuple: ("(", ")"),
        set: ("set([", "])"),
        frozenset: ("frozenset([", "])"),
        }

    #: Filesystem path to the Python standard library
    _stdlib_path = os.path.dirname(os.path.realpath(inspect.getsourcefile(os)))

    def __init__(self, etype, value, tb, **options):
        """
        :param etype: The exception type
        :type etype: type
        :param value: The exception instance
        :type value: Exception
        :param tb: The traceback instance
        :type tb: traceback
        :param options: Options for this instance
        :type options: dict
        """

        #: The exception type
        self.etype = etype

        #: The exception instance
        self.value = value

        self.options = XTracebackOptions(**options)


        # placeholders
        self._lexer = None
        self._formatter = None

        # keep track of objects we've seen
        self.seen = {}

        # get the traceback frames and work out number padding
        self.tb_frames = []
        self.number_padding = 0
        i = 0
        while tb is not None and (self.options.limit is None
                                  or i < self.options.limit):
            if i >= self.options.offset:
                try:
                    frame_info = inspect.getframeinfo(tb, self.options.context)
                except KeyError:
                    # <stdlib>/inspect.py line 506 - there may be no __main__
                    pass
                frame = XTracebackFrame(self, tb.tb_frame, frame_info, i)
                if not frame.exclude:
                    self.tb_frames.append(frame)
                    self.number_padding = max(len(str(frame_info[1])),
                                              self.number_padding)
            tb = tb.tb_next
            i += 1

        # get rid of tb once we no longer need it
        tb = None

    @property
    def tty_stream(self):
        """
        Whether or not our stream is a tty
        """
        return hasattr(self.options.stream, "isatty") \
            and self.options.stream.isatty()

    @property
    def color(self):
        """
        Whether or not color should be output
        """
        return self.tty_stream if self.options.color is None \
            else self.options.color

    @property
    def print_width(self):
        """
        Width of one screen for line width calculations

        If not explicitly set in self.options the width will be the terminal
        width if on Unix.
        """
        print_width = self.options.print_width
        if print_width is None \
            and fcntl is not None \
            and self.tty_stream:
            print_width = struct.unpack(
                'HHHH',
                fcntl.ioctl(self.options.stream,
                            termios.TIOCGWINSZ,
                            struct.pack('HHHH', 0, 0, 0, 0)),
                )[1]
        else:
            print_width = self.DEFAULT_WIDTH
        return print_width

    def __str__(self):
        return self._str_lines(self._format_exception())

    def _format_filename(self, filename):
        if self.options.shorten_filenames:
            filename = os.path.realpath(filename)
            if filename.startswith(self._stdlib_path):
                filename = filename.replace(self._stdlib_path, "<stdlib>")
            elif hasattr(os.path, "relpath"):
                # os.path.relpath was introduced in python 2.5
                relative = os.path.relpath(filename)
                if len(relative) < len(filename):
                    filename = relative
        # using str on filename to make Jython (default string is unicode)
        # consistent with CPython
        return str(filename)

    def _format_variable(self, key, value, indent=4, prefix="", separator=" = "):
        base_size = indent + len(prefix) + len(key) + len(separator)
        if isinstance(value, basestring) and len(value) > self.print_width * 2:
            # truncate long strings - minus 2 for the quotes and 3 for
            # the ellipsis
            value = value[:self.print_width - base_size - 2 - 3] + "..."
        vtype = type(value)
        try:
            pvalue = pprint.pformat(value, indent=0)
        except:
            pvalue = "<unprintable %s object>" % vtype.__name__
        if base_size + len(pvalue) > self.print_width:
            reformat = self.REFORMAT.get(vtype)
            if reformat is not None:
                start, end = reformat
                lines = map(str.strip,
                            pvalue.lstrip(start).rstrip(end).splitlines())
                sub_indent = "\n" + " " * (indent + 4)
                pvalue = "".join((start, sub_indent, sub_indent.join(lines),
                                  ",", sub_indent, end))
        return "".join((" " * indent, prefix, key, separator, pvalue))

    # { Line formatting

    def _highlight(self, string):
        if pygments is None:
            warnings.warn("highlighting not available - pygments is required")
        else:
            if self._lexer is None:
                self._lexer = PythonXTracebackLexer()
            if self._formatter is None:
                try:
                    self._formatter = TerminalFormatter()
                except pygments.util.ClassNotFound, exc:
                    warnings.warn("highlighting not available - %s" % exc)
                    # this can happen when running under gae
                    return string
            try:<commit_msg>remove app engine hack as unnecessary<commit_after>import inspect
import os
import pprint
import struct
import warnings

try:
    import fcntl
except ImportError:
    fcntl = None
else:
    import termios

try:
    import pygments
except ImportError:
    pygments = None
else:
    from pygments.formatters.terminal import TerminalFormatter
    from .lexer import PythonXTracebackLexer

from .xtracebackframe import XTracebackFrame
from .xtracebackoptions import XTracebackOptions


class XTraceback(object):
    """
    An extended traceback formatter

    """

    #: The default width of one line
    DEFAULT_WIDTH = 80

    #: A dictionary keyed by type with values as the wrapper for string
    #: representation of the type
    REFORMAT = {
        dict: ("{", "}"),
        list: ("[", "]"),
        tuple: ("(", ")"),
        set: ("set([", "])"),
        frozenset: ("frozenset([", "])"),
        }

    #: Filesystem path to the Python standard library
    _stdlib_path = os.path.dirname(os.path.realpath(inspect.getsourcefile(os)))

    def __init__(self, etype, value, tb, **options):
        """
        :param etype: The exception type
        :type etype: type
        :param value: The exception instance
        :type value: Exception
        :param tb: The traceback instance
        :type tb: traceback
        :param options: Options for this instance
        :type options: dict
        """

        #: The exception type
        self.etype = etype

        #: The exception instance
        self.value = value

        self.options = XTracebackOptions(**options)


        # placeholders
        self._lexer = None
        self._formatter = None

        # keep track of objects we've seen
        self.seen = {}

        # get the traceback frames and work out number padding
        self.tb_frames = []
        self.number_padding = 0
        i = 0
        while tb is not None and (self.options.limit is None
                                  or i < self.options.limit):
            if i >= self.options.offset:
                try:
                    frame_info = inspect.getframeinfo(tb, self.options.context)
                except KeyError:
                    # <stdlib>/inspect.py line 506 - there may be no __main__
                    pass
                frame = XTracebackFrame(self, tb.tb_frame, frame_info, i)
                if not frame.exclude:
                    self.tb_frames.append(frame)
                    self.number_padding = max(len(str(frame_info[1])),
                                              self.number_padding)
            tb = tb.tb_next
            i += 1

        # get rid of tb once we no longer need it
        tb = None

    @property
    def tty_stream(self):
        """
        Whether or not our stream is a tty
        """
        return hasattr(self.options.stream, "isatty") \
            and self.options.stream.isatty()

    @property
    def color(self):
        """
        Whether or not color should be output
        """
        return self.tty_stream if self.options.color is None \
            else self.options.color

    @property
    def print_width(self):
        """
        Width of one screen for line width calculations

        If not explicitly set in self.options the width will be the terminal
        width if on Unix.
        """
        print_width = self.options.print_width
        if print_width is None \
            and fcntl is not None \
            and self.tty_stream:
            print_width = struct.unpack(
                'HHHH',
                fcntl.ioctl(self.options.stream,
                            termios.TIOCGWINSZ,
                            struct.pack('HHHH', 0, 0, 0, 0)),
                )[1]
        else:
            print_width = self.DEFAULT_WIDTH
        return print_width

    def __str__(self):
        return self._str_lines(self._format_exception())

    def _format_filename(self, filename):
        if self.options.shorten_filenames:
            filename = os.path.realpath(filename)
            if filename.startswith(self._stdlib_path):
                filename = filename.replace(self._stdlib_path, "<stdlib>")
            elif hasattr(os.path, "relpath"):
                # os.path.relpath was introduced in python 2.5
                relative = os.path.relpath(filename)
                if len(relative) < len(filename):
                    filename = relative
        # using str on filename to make Jython (default string is unicode)
        # consistent with CPython
        return str(filename)

    def _format_variable(self, key, value, indent=4, prefix="", separator=" = "):
        base_size = indent + len(prefix) + len(key) + len(separator)
        if isinstance(value, basestring) and len(value) > self.print_width * 2:
            # truncate long strings - minus 2 for the quotes and 3 for
            # the ellipsis
            value = value[:self.print_width - base_size - 2 - 3] + "..."
        vtype = type(value)
        try:
            pvalue = pprint.pformat(value, indent=0)
        except:
            pvalue = "<unprintable %s object>" % vtype.__name__
        if base_size + len(pvalue) > self.print_width:
            reformat = self.REFORMAT.get(vtype)
            if reformat is not None:
                start, end = reformat
                lines = map(str.strip,
                            pvalue.lstrip(start).rstrip(end).splitlines())
                sub_indent = "\n" + " " * (indent + 4)
                pvalue = "".join((start, sub_indent, sub_indent.join(lines),
                                  ",", sub_indent, end))
        return "".join((" " * indent, prefix, key, separator, pvalue))

    # { Line formatting

    def _highlight(self, string):
        if pygments is None:
            warnings.warn("highlighting not available - pygments is required")
        else:
            if self._lexer is None:
                self._lexer = PythonXTracebackLexer()
            if self._formatter is None:
                self._formatter = TerminalFormatter()
            try:
{'fraction_non_alphanumeric': 0.09092382495948136, 'fraction_numerical': 0.0038897893030794164, 'mean_word_length': 1.7081413210445469, 'pattern_counts': {'":': 0, '<': 13, '<?xml version=': 0, '>': 15, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 8, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7116759', 'n_tokens_mistral': 3454, 'n_tokens_neox': 3239, 'n_words': 974}