<commit_before>import cStringIO
import json
import logging
import sys
import traceback
import datetime
import decimal

logging.basicConfig()
logger = logging.getLogger('fake_shell')

sys_stdin = sys.stdin
sys_stdout = sys.stdout
sys_stderr = sys.stderr

fake_stdin = cStringIO.StringIO()
fake_stdout = cStringIO.StringIO()
fake_stderr = cStringIO.StringIO()

sys.stdin = fake_stdin
sys.stdout = fake_stdout
sys.stderr = fake_stderr

global_dict = {}

execution_count = 0


def execute_reply(status, content):
    global execution_count
    execution_count += 1

    return {
        'msg_type': 'execute_reply',
        'content': dict(
            content,
            status=status,
            execution_count=execution_count - 1
        )
    }

def execute_reply_ok(data):
    return execute_reply('ok', {
        'data': data,
    })

def execute_reply_error(exc_type, exc_value, tb):
    logger.error('execute_reply', exc_info=True)
    return execute_reply('error', {
        'ename': unicode(exc_type.__name__),
        'evalue': unicode(exc_value),
        'traceback': traceback.format_exception(exc_type, exc_value, tb, -1),
    })


def execute(code):
    try:
        code = compile(code, '<stdin>', 'single')
        exec code in global_dict
    except:
        return execute_reply_error(*sys.exc_info())

    stdout = fake_stdout.getvalue()
    stderr = fake_stderr.getvalue()

    output = ''

    if stdout:
        output += stdout
<commit_msg>[livy] Allow multiline python cells<commit_after>import ast
import cStringIO
import datetime
import decimal
import json
import logging
import sys
import traceback

logging.basicConfig()
logger = logging.getLogger('fake_shell')

sys_stdin = sys.stdin
sys_stdout = sys.stdout
sys_stderr = sys.stderr

fake_stdin = cStringIO.StringIO()
fake_stdout = cStringIO.StringIO()
fake_stderr = cStringIO.StringIO()

sys.stdin = fake_stdin
sys.stdout = fake_stdout
sys.stderr = fake_stderr

global_dict = {}

execution_count = 0


def execute_reply(status, content):
    global execution_count
    execution_count += 1

    return {
        'msg_type': 'execute_reply',
        'content': dict(
            content,
            status=status,
            execution_count=execution_count - 1
        )
    }

def execute_reply_ok(data):
    return execute_reply('ok', {
        'data': data,
    })

def execute_reply_error(exc_type, exc_value, tb):
    logger.error('execute_reply', exc_info=True)
    return execute_reply('error', {
        'ename': unicode(exc_type.__name__),
        'evalue': unicode(exc_value),
        'traceback': traceback.format_exception(exc_type, exc_value, tb, -1),
    })


def execute(code):
    try:
        code = ast.parse(code)
        to_run_exec, to_run_single = code.body[:-1], code.body[-1:]

        for node in to_run_exec:
            mod = ast.Module([node])
            code = compile(mod, '<stdin>', 'exec')
            exec code in global_dict

        for node in to_run_single:
            mod = ast.Interactive([node])
            code = compile(mod, '<stdin>', 'single')
            exec code in global_dict
    except:
        return execute_reply_error(*sys.exc_info())

    stdout = fake_stdout.getvalue()
    stderr = fake_stderr.getvalue()

    output = ''

    if stdout:
        output += stdout

{'fraction_non_alphanumeric': 0.10835351089588378, 'fraction_numerical': 0.003026634382566586, 'mean_word_length': 2.9867310012062727, 'pattern_counts': {'":': 0, '<': 6, '<?xml version=': 0, '>': 6, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '528949', 'n_tokens_mistral': 1057, 'n_tokens_neox': 1024, 'n_words': 232}