<commit_before>import re
import types
from .. import logger

class PPError(Exception):
    pass

directive_re = re.compile('^#py.')

class Context(object):
    """ Context """

    filename = None
    globalz = {}

    def __init__(self, filename):
        self.filename = filename
        self.exec_stmt('import sys, os')
        self.exec_stmt('_PYASPP_ROOT = os.path.abspath(os.path.dirname(sys.argv[0]))')
        self.eval_expr('sys.path.append(_PYASPP_ROOT + "/pyaspp/pp" )')
        self.exec_stmt('import cpp')

    def eval_expr(self, expr):
        return eval(expr, self.globalz)

    def exec_stmt(self, stmt):
        exec(stmt, self.globalz)

_py_block = []
_in_py_block = False

def exec_stmt(ctx, stmt):
    ctx.exec_stmt(stmt.lstrip())

def eval_expr(ctx, expr):
    ret = ctx.eval_expr(expr)
    if hasattr(ret, '__str__'):
        print(str(ret))

def begin_block(ctx, expr):
    global _in_py_block
    if _in_py_block:
        raise PPError('nested py block are not allowed!')
    _in_py_block = True
    _py_block = []

def end_block(ctx, expr):
    global _in_py_block
    if not _in_py_block:
        raise PPError('closing a non-opened py block!')
    _in_py_block = False
    ctx.exec_stmt('\n'.join(_py_block))<commit_msg>Now supports yield expressions (generators)<commit_after>import re
import types
from .. import logger

class PPError(Exception):
    pass

directive_re = re.compile('^#py.')

class Context(object):
    """ Context """

    filename = None
    globalz = {}

    def __init__(self, filename):
        self.filename = filename
        self.exec_stmt('import sys, os')
        self.exec_stmt('_PYASPP_ROOT = os.path.abspath(os.path.dirname(sys.argv[0]))')
        self.eval_expr('sys.path.append(_PYASPP_ROOT + "/pyaspp/pp" )')
        self.exec_stmt('import cpp')

    def eval_expr(self, expr):
        return eval(expr, self.globalz)

    def exec_stmt(self, stmt):
        exec(stmt, self.globalz)

_py_block = []
_in_py_block = False

def exec_stmt(ctx, stmt):
    ctx.exec_stmt(stmt.lstrip())

def eval_expr(ctx, expr):
    ret = ctx.eval_expr(expr)
    if ret is None:
        return
    if isinstance(ret, types.GeneratorType):
        for step in ret:
            print(str(step))
    elif hasattr(ret, '__str__'):
        print(str(ret))

def begin_block(ctx, expr):
    global _in_py_block
    if _in_py_block:
        raise PPError('nested py block are not allowed!')
    _in_py_block = True
    _py_block = []

def end_block(ctx, expr):
    global _in_py_block
    if not _in_py_block:
        raise PPError('closing a non-opened py block!')
    _in_py_block = False
    ctx.exec_stmt('\n'.join(_py_block))
{'fraction_non_alphanumeric': 0.12575075075075076, 'fraction_numerical': 0.0007507507507507507, 'mean_word_length': 3.0440060698027316, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2177579', 'n_tokens_mistral': 939, 'n_tokens_neox': 933, 'n_words': 202}