<commit_before>"""Beautiful and helpful exceptions

Just `import better_exceptions` somewhere. It handles the rest.


   Name: better_exceptions
 Author: <NAME>
  Email: <EMAIL>
    URL: github.com/qix-/better-exceptions
License: Copyright (c) 2017 <NAME>, licensed under the MIT licens
"""

from __future__ import absolute_import
from __future__ import print_function

import _ast
import ast
import inspect
import keyword
import os
import re
import sys
import traceback


def isast(v):
    return inspect.isclass(v) and issubclass(v, ast.AST)


NOCOLOR = not os.isatty(2) or os.name == 'nt' or os.getenv('TERM', '')[:5] != 'xterm'

COMMENT_REGXP = re.compile(r'((?:(?:"(?:[^\\"]|(\\\\)*\\")*")|(?:\'(?:[^\\"]|(\\\\)*\\\')*\')|[^#])*)(#.*)$')

AST_ELEMENTS = {
    'builtins': __builtins__.keys(),
    'keywords': [getattr(ast, cls) for cls in dir(ast) if keyword.iskeyword(cls.lower()) and isast(getattr(ast, cls))],
}

THEME = {
    'comment': lambda s: '\x1b[2;37m{}\x1b[m'.format(s),
    'keyword': lambda s: '\x1b[33;1m{}\x1b[m'.format(s),
    'builtin': lambda s: '\x1b[35;1m{}\x1b[m'.format(s),
    'literal': lambda s: '\x1b[31m{}\x1b[m'.format(s),
    'inspect': lambda s: s if NOCOLOR else u'\x1b[36m{}\x1b[m'.format(s),
}

MAX_LENGTH = 128


def colorize_comment(source):
    match = COMMENT_REGXP.match(source)
    if match:
        source = '{}{}'.format(match.group(1), THEME['comment'](match.group(4)))
    return source


def colorize_tree(tree, source):
    if NOCOLOR:
        # quick fail
        return source

    chunks = []

    offset = 0
    nodes = [n for n in ast.walk(tree)]
    nnodes = len(nodes)

    def append(offset, node, s, theme):
        begin_col = node.col_offset
        chunks.append(source[offset:begin_col])
        chunks.append(THEME[theme](s))
        return begin_col + len(s)

    for i in xrange(nnodes):
        node = nodes[i]
        nodecls = node.__class__
        nodename = nodecls.__name__

        if 'col_offset' not in dir(node):
            # this would probably benefit from using the `parser` module in the future...
            continue

        if nodecls in AST_ELEMENTS['keywords']:
            offset = append(offset, node, nodename.lower(), 'keyword')

        if nodecls == ast.Name and node.id in AST_ELEMENTS['builtins']:
            offset = append(offset, node, node.id, 'builtin')

        if nodecls == ast.Str:
            offset = append(offset, node, "'{}'".format(node.s), 'literal')

        if nodecls == ast.Num:
            offset = append(offset, node, str(node.n), 'literal')

    chunks.append(source[offset:])
    return colorize_comment(''.join(chunks))


def get_source_line(frame):
    line = frame.f_lineno
    try:
        filename = inspect.getsourcefile(frame)
    except TypeError:
        return None

    count = 1
    with open(frame.f_code.co_filename, 'r') as f:
        while count < line:
            next(f)
            count += 1
        return (filename, line, next(f).strip())


def get_relevant_names(source, tree):
    return [node for node in ast.walk(tree) if isinstance(node, ast.Name)]


def format_value(v):
    v = repr(v)
    if MAX_LENGTH is not None and len(v) > MAX_LENGTH:
        v = v[:MAX_LENGTH] + '...'
    return v


def get_relevant_values(source, frame, tree):
    names = get_relevant_names(source, tree)
    values = []

    for name in names:
        text = name.id
        col = name.col_offset
        if frame.f_locals.has_key(text):
            val = frame.f_locals.get(text, None)
            values.append((text, col, format_value(val)))
        elif frame.f_globals.has_key(text):
            val = frame.f_globals.get(text, None)
            values.append((text, col, format_value(val)))<commit_msg>sort values by column position<commit_after>"""Beautiful and helpful exceptions

Just `import better_exceptions` somewhere. It handles the rest.


   Name: better_exceptions
 Author: <NAME>
  Email: <EMAIL>
    URL: github.com/qix-/better-exceptions
License: Copyright (c) 2017 <NAME>, licensed under the MIT licens
"""

from __future__ import absolute_import
from __future__ import print_function

import _ast
import ast
import inspect
import keyword
import os
import re
import sys
import traceback


def isast(v):
    return inspect.isclass(v) and issubclass(v, ast.AST)


NOCOLOR = not os.isatty(2) or os.name == 'nt' or os.getenv('TERM', '')[:5] != 'xterm'

COMMENT_REGXP = re.compile(r'((?:(?:"(?:[^\\"]|(\\\\)*\\")*")|(?:\'(?:[^\\"]|(\\\\)*\\\')*\')|[^#])*)(#.*)$')

AST_ELEMENTS = {
    'builtins': __builtins__.keys(),
    'keywords': [getattr(ast, cls) for cls in dir(ast) if keyword.iskeyword(cls.lower()) and isast(getattr(ast, cls))],
}

THEME = {
    'comment': lambda s: '\x1b[2;37m{}\x1b[m'.format(s),
    'keyword': lambda s: '\x1b[33;1m{}\x1b[m'.format(s),
    'builtin': lambda s: '\x1b[35;1m{}\x1b[m'.format(s),
    'literal': lambda s: '\x1b[31m{}\x1b[m'.format(s),
    'inspect': lambda s: s if NOCOLOR else u'\x1b[36m{}\x1b[m'.format(s),
}

MAX_LENGTH = 128


def colorize_comment(source):
    match = COMMENT_REGXP.match(source)
    if match:
        source = '{}{}'.format(match.group(1), THEME['comment'](match.group(4)))
    return source


def colorize_tree(tree, source):
    if NOCOLOR:
        # quick fail
        return source

    chunks = []

    offset = 0
    nodes = [n for n in ast.walk(tree)]
    nnodes = len(nodes)

    def append(offset, node, s, theme):
        begin_col = node.col_offset
        chunks.append(source[offset:begin_col])
        chunks.append(THEME[theme](s))
        return begin_col + len(s)

    for i in xrange(nnodes):
        node = nodes[i]
        nodecls = node.__class__
        nodename = nodecls.__name__

        if 'col_offset' not in dir(node):
            # this would probably benefit from using the `parser` module in the future...
            continue

        if nodecls in AST_ELEMENTS['keywords']:
            offset = append(offset, node, nodename.lower(), 'keyword')

        if nodecls == ast.Name and node.id in AST_ELEMENTS['builtins']:
            offset = append(offset, node, node.id, 'builtin')

        if nodecls == ast.Str:
            offset = append(offset, node, "'{}'".format(node.s), 'literal')

        if nodecls == ast.Num:
            offset = append(offset, node, str(node.n), 'literal')

    chunks.append(source[offset:])
    return colorize_comment(''.join(chunks))


def get_source_line(frame):
    line = frame.f_lineno
    try:
        filename = inspect.getsourcefile(frame)
    except TypeError:
        return None

    count = 1
    with open(frame.f_code.co_filename, 'r') as f:
        while count < line:
            next(f)
            count += 1
        return (filename, line, next(f).strip())


def get_relevant_names(source, tree):
    return [node for node in ast.walk(tree) if isinstance(node, ast.Name)]


def format_value(v):
    v = repr(v)
    if MAX_LENGTH is not None and len(v) > MAX_LENGTH:
        v = v[:MAX_LENGTH] + '...'
    return v


def get_relevant_values(source, frame, tree):
    names = get_relevant_names(source, tree)
    values = []

    for name in names:
        text = name.id
        col = name.col_offset
        if frame.f_locals.has_key(text):
            val = frame.f_locals.get(text, None)
            values.append((text, col, format_value(val)))
        elif frame.f_globals.has_key(text):
            val = frame.f_globals.get(text, None)
            values.append((text, col, format_value(val)))

    values.sort(key=lambda e: e[1])
{'fraction_non_alphanumeric': 0.1530801911842804, 'fraction_numerical': 0.009957514604354753, 'mean_word_length': 2.8512269938650308, 'pattern_counts': {'":': 0, '<': 11, '<?xml version=': 0, '>': 11, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3282976', 'n_tokens_mistral': 2639, 'n_tokens_neox': 2541, 'n_words': 634}