<commit_before>import os
import re
import ast
import imp
import inspect

from IPython.core.magic import register_cell_magic
from IPython.display import display
    
    
__all__ = ['show_ast', 'show_source', 'show_text', 'Settings', 'Renderers',]


RENDERING_PATH = os.path.join(os.path.dirname(__file__), 'rendering')


class Renderers:
    graphviz = 'graphviz', [RENDERING_PATH]
    nltk = 'nltk', [RENDERING_PATH]
    

Settings = dict(
    # Styling options:
    scale=2,
    font='courier',
    shape='none',
    terminal_color='#008040',
    nonterminal_color='#004080',

    # AST display options:
    omit_module=True,
    omit_docstrings=True,
    omit_location_info=True,
    locations_format="L{begin[0]}:{begin[1]}-L{end[0]}:{end[1]}",

    # Rendering engine is expected to expose "render" function
    renderer=Renderers.graphviz,
)
        
        
def show_ast(module, settings=Settings):
    omit_docstrings = settings['omit_docstrings']
    if settings['omit_module'] and len(module.body) == 1:
        node = module.body[0]
    else:
        node = module
    renderer = imp.load_module(
        'renderer', 
        *imp.find_module(*settings['renderer'])
    )
    display(renderer.render(node, settings))


@register_cell_magic
def showast(__, cell):
    m = ast.parse(cell)
    show_ast(m)


def show_source(item, settings=Settings):
    src = inspect.getsource(item)
    show_text(src, settings)

def show_text(src, settings=Settings):
    try:<commit_msg>extra space<commit_after>import os
import re
import ast
import imp
import inspect

from IPython.core.magic import register_cell_magic
from IPython.display import display
    
    
__all__ = ['show_ast', 'show_source', 'show_text', 'Settings', 'Renderers',]


RENDERING_PATH = os.path.join(os.path.dirname(__file__), 'rendering')


class Renderers:
    graphviz = 'graphviz', [RENDERING_PATH]
    nltk = 'nltk', [RENDERING_PATH]
    

Settings = dict(
    # Styling options:
    scale=2,
    font='courier',
    shape='none',
    terminal_color='#008040',
    nonterminal_color='#004080',

    # AST display options:
    omit_module=True,
    omit_docstrings=True,
    omit_location_info=True,
    locations_format="L{begin[0]}:{begin[1]}-L{end[0]}:{end[1]}",

    # Rendering engine is expected to expose "render" function
    renderer=Renderers.graphviz,
)
        
        
def show_ast(module, settings=Settings):
    omit_docstrings = settings['omit_docstrings']
    if settings['omit_module'] and len(module.body) == 1:
        node = module.body[0]
    else:
        node = module
    renderer = imp.load_module(
        'renderer', 
        *imp.find_module(*settings['renderer'])
    )
    display(renderer.render(node, settings))


@register_cell_magic
def showast(__, cell):
    m = ast.parse(cell)
    show_ast(m)


def show_source(item, settings=Settings):
    src = inspect.getsource(item)
    show_text(src, settings)


def show_text(src, settings=Settings):
    try:
{'fraction_non_alphanumeric': 0.12689841376982788, 'fraction_numerical': 0.01282483968950388, 'mean_word_length': 3.6973058637083995, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2442602', 'n_tokens_mistral': 1051, 'n_tokens_neox': 1007, 'n_words': 198}