<commit_before>import os
import sys
import atexit
import readline
import traceback
from copy import deepcopy
from django.template import Parser, Lexer, Context, TemplateSyntaxError

def run_shell(context=Context()):
    goodbye = lambda: sys.stderr.write('\nkthxbai!\n')
    while True:
        try:
            for node in input_node_generator():
                sys.stdout.write(node.render(context))
            print
        except KeyboardInterrupt:
            goodbye()
            break
        except EOFError:
            goodbye()
            break
        except Exception, e:
            traceback.print_exc()

def input_node_generator(prompt='>>> ', leading_tokens=None, input_source=raw_input):
    input = False
    while not input:
        input = input_source(prompt)
    tokens = Lexer(input, None).tokenize()
    if leading_tokens:
        tokens = leading_tokens + tokens
    initial_tokens = deepcopy(tokens)
    try:
        for node in Parser(tokens).parse():
            yield node
    except TemplateSyntaxError, e:
        if e.args[0].startswith('Unclosed tags'):
            for node in input_node_generator('... ', initial_tokens):
                yield node
        else:
            raise

def setup_readline_history():
    history_path = os.path.join(os.getenv('HOME'), '.django-template-repl-history')<commit_msg>This adds line endings to the template so the lines don't get squashed together.<commit_after>import os
import sys
import atexit
import readline
import traceback
from copy import deepcopy
from django.template import Parser, Lexer, Context, TemplateSyntaxError

def run_shell(context=Context()):
    goodbye = lambda: sys.stderr.write('\nkthxbai!\n')
    while True:
        try:
            for node in input_node_generator():
                sys.stdout.write(node.render(context))
            print
        except KeyboardInterrupt:
            goodbye()
            break
        except EOFError:
            goodbye()
            break
        except Exception, e:
            traceback.print_exc()

def input_node_generator(prompt='>>> ', leading_tokens=None, input_source=raw_input):
    input = False
    while not input:
        input = input_source(prompt)
    input = input + '\n'
    tokens = Lexer(input, None).tokenize()
    if leading_tokens:
        tokens = leading_tokens + tokens
    initial_tokens = deepcopy(tokens)
    try:
        for node in Parser(tokens).parse():
            yield node
    except TemplateSyntaxError, e:
        if e.args[0].startswith('Unclosed tags'):
            for node in input_node_generator('... ', initial_tokens):
                yield node
        else:
            raise

def setup_readline_history():
    history_path = os.path.join(os.getenv('HOME'), '.django-template-repl-history')
{'fraction_non_alphanumeric': 0.09296482412060302, 'fraction_numerical': 0.0007178750897343862, 'mean_word_length': 2.4280442804428044, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2612501', 'n_tokens_mistral': 771, 'n_tokens_neox': 758, 'n_words': 205}