<commit_before>import ast
from collections import Iterable, Sequence, MutableSequence
from functools import reduce
from pathlib import Path
from os import chdir
from os.path import normpath, abspath

from mochi import GE_PYTHON_34, IS_PYPY
from mochi.parser import Symbol, Keyword, parse, lex, get_temp_name
from .utils import issequence_except_str
from .constants import *
from .exceptions import MochiSyntaxError, DuplicatedDefError
from .global_env import global_env


if GE_PYTHON_34:
    expr_and_stmt_ast = (ast.Expr, ast.If, ast.For, ast.FunctionDef, ast.Assign, ast.Delete, ast.Try, ast.Raise,
                         ast.With, ast.While, ast.Break, ast.Return, ast.Continue, ast.ClassDef,
                         ast.Import, ast.ImportFrom, ast.Pass)
else:
    expr_and_stmt_ast = (ast.Expr, ast.If, ast.For, ast.FunctionDef, ast.Assign, ast.Delete, ast.TryFinally,
                         ast.TryExcept, ast.Raise, ast.With, ast.While, ast.Break, ast.Return, ast.Continue,
                         ast.ClassDef, ast.Import, ast.ImportFrom, ast.Pass)

syntax_table = {}


#-- double builtins :(
def chunks(l, n):
    for i in range(0, len(l), n):
        yield l[i:i + n]

def tuple_it(obj):
    return tuple(map(tuple_it, obj)) if isinstance(obj, MutableSequence) else obj

#---

def syntax(*names):
    def _syntax(func):
        for name in names:
            syntax_table[name] = func
            if name not in global_env.keys():
                global_env[name] = 'syntax_' + name
        return func

    return _syntax

required_filenames_stack = [set()]


def lexical_scope_for_require(func):
    def _lexical_scope(*args, **kwargs):
        try:
            required_filenames_stack.append(set())
            return func(*args, **kwargs)
        finally:
            required_filenames_stack.pop()
    return _lexical_scope


def lexical_scope(quasi=False):
    def _lexical_scope(func):
        def __lexical_scope(*args, **kwargs):
            try:
                scope = Scope(quasi)
                if quasi:
                    binding_name_set_stack[-1].add_child(scope)
                binding_name_set_stack.append(scope)
                return func(*args, **kwargs)
            finally:
                if binding_name_set_stack[-1] is scope:
                    binding_name_set_stack.pop()

        return __lexical_scope

    return _lexical_scope


def flatten_list(lis):
    i = 0
    while i < len(lis):
        while isinstance(lis[i], Iterable):
            if not lis[i]:
                lis.pop(i)
                i -= 1
                break
            else:
                lis[i:i + 1] = lis[i]
        i += 1
    return lis


def add_binding_name(binding_name, file_name):
    return binding_name_set_stack[-1].add_binding_name(binding_name, file_name)


class Translator(object):
    def __init__(self, macro_table={}, filename='<string>'):
        self.hidden_vars = []
        self.macro_table = macro_table
        self.filename = filename

    def translate_file(self, filename, show_tokens=False):
        body = []
        self.filename = filename
        with open(filename, 'r') as f:
            sexps = parse(lex(f.read(), debug=show_tokens), filename)
            chdir(normpath(str(Path(filename).parent)))
            for sexp in sexps:
                if isinstance(sexp, MutableSequence):
                    sexp = tuple_it(sexp)
                if sexp is COMMENT:
                    continue
                pre, value = self.translate(sexp)
                body.extend([self.enclose(exp, True) for exp in pre])
                body.append(self.enclose(value, True))
        return ast.Module(body=body)

    def translate_loaded_file(self, filename, show_tokens=False):
        body = []
        self.filename = filename
        with open(filename, 'r') as f:
            sexps = parse(lex(f.read(), debug=show_tokens), filename)
            for sexp in sexps:
                if isinstance(sexp, MutableSequence):
                    sexp = tuple_it(sexp)
                if sexp is COMMENT:
                    continue
                pre, value = self.translate(sexp)
                body.extend(pre)
                body.append(value)
        return body

    def translate_required_file(self, filename):
        required_filenames = required_filenames_stack[-1]
        if filename in required_filenames:
            return self.translate_ref(NONE_SYM)[1],
        else:
            required_filenames.add(filename)
            return self.translate_loaded_file(filename)

    def translate_loaded_files(self, file_paths):
        body = []
        for file_path in file_paths:
            if not isinstance(file_path, str):
                raise MochiSyntaxError(file_path, self.filename)
            body.extend(self.translate_loaded_file(abspath(file_path)))
        return body, self.translate_ref(NONE_SYM)[1]

    def translate_required_files(self, file_paths):
        body = []
        for file_path in file_paths:
            if not isinstance(file_path, str):
                raise MochiSyntaxError(file_path, self.filename)
            body.extend(self.translate_required_file(abspath(file_path)))
        return body, self.translate_ref(NONE_SYM)[1]

    def translate_sexp_to_interact(self, sexp):
        pre, value = self.translate(sexp, False)
        body = [self.enclose(exp, True) for exp in pre]
        write_value = ast.Call(func=ast.Name(id='write',
                                             ctx=ast.Load(),
                                             lineno=1,
                                             col_offset=0),
                               ctx=ast.Load(),
                               args=[value],
                               keywords=[],
                               starargs=None,
                               kwargs=None,
                               lineno=1,
                               col_offset=0)
        body.append(self.enclose(write_value, True))
        return ast.Module(body=body)

    def translate_sexp(self, sexp):
        # TODO value
        pre, value = self.translate(sexp, False)
        body = [self.enclose(exp, True) for exp in pre]
        return ast.Module(body=body)

    def is_self_evaluating(self, exp):
        return isinstance(exp, (int, float, complex, str, bool, type(None), Keyword))

    def translate_self_evaluating(self, exp):
        if isinstance(exp, (int, float, complex)):
            if hasattr(exp, 'value'):
                return EMPTY, ast.Num(exp.value,
                                      lineno=exp.lineno,
                                      col_offset=0)
            else:<commit_msg>Add translation of sexpressions from string.<commit_after>import ast
from collections import Iterable, Sequence, MutableSequence
from functools import reduce
from pathlib import Path
from os import chdir
from os.path import normpath, abspath

from mochi import GE_PYTHON_34, IS_PYPY
from mochi.parser import Symbol, Keyword, parse, lex, get_temp_name
from .utils import issequence_except_str
from .constants import *
from .exceptions import MochiSyntaxError, DuplicatedDefError
from .global_env import global_env


if GE_PYTHON_34:
    expr_and_stmt_ast = (ast.Expr, ast.If, ast.For, ast.FunctionDef, ast.Assign, ast.Delete, ast.Try, ast.Raise,
                         ast.With, ast.While, ast.Break, ast.Return, ast.Continue, ast.ClassDef,
                         ast.Import, ast.ImportFrom, ast.Pass)
else:
    expr_and_stmt_ast = (ast.Expr, ast.If, ast.For, ast.FunctionDef, ast.Assign, ast.Delete, ast.TryFinally,
                         ast.TryExcept, ast.Raise, ast.With, ast.While, ast.Break, ast.Return, ast.Continue,
                         ast.ClassDef, ast.Import, ast.ImportFrom, ast.Pass)

syntax_table = {}


#-- double builtins :(
def chunks(l, n):
    for i in range(0, len(l), n):
        yield l[i:i + n]

def tuple_it(obj):
    return tuple(map(tuple_it, obj)) if isinstance(obj, MutableSequence) else obj

#---

def syntax(*names):
    def _syntax(func):
        for name in names:
            syntax_table[name] = func
            if name not in global_env.keys():
                global_env[name] = 'syntax_' + name
        return func

    return _syntax

required_filenames_stack = [set()]


def lexical_scope_for_require(func):
    def _lexical_scope(*args, **kwargs):
        try:
            required_filenames_stack.append(set())
            return func(*args, **kwargs)
        finally:
            required_filenames_stack.pop()
    return _lexical_scope


def lexical_scope(quasi=False):
    def _lexical_scope(func):
        def __lexical_scope(*args, **kwargs):
            try:
                scope = Scope(quasi)
                if quasi:
                    binding_name_set_stack[-1].add_child(scope)
                binding_name_set_stack.append(scope)
                return func(*args, **kwargs)
            finally:
                if binding_name_set_stack[-1] is scope:
                    binding_name_set_stack.pop()

        return __lexical_scope

    return _lexical_scope


def flatten_list(lis):
    i = 0
    while i < len(lis):
        while isinstance(lis[i], Iterable):
            if not lis[i]:
                lis.pop(i)
                i -= 1
                break
            else:
                lis[i:i + 1] = lis[i]
        i += 1
    return lis


def add_binding_name(binding_name, file_name):
    return binding_name_set_stack[-1].add_binding_name(binding_name, file_name)


class Translator(object):
    def __init__(self, macro_table={}, filename='<string>'):
        self.hidden_vars = []
        self.macro_table = macro_table
        self.filename = filename

    def translate_sexps(self, sexps):
        """Translate sexpressions into a Python AST.
        """
        body = []
        for sexp in sexps:
            if isinstance(sexp, MutableSequence):
                sexp = tuple_it(sexp)
            if sexp is COMMENT:
                continue
            pre, value = self.translate(sexp)
            body.extend([self.enclose(exp, True) for exp in pre])
            body.append(self.enclose(value, True))
        return ast.Module(body=body)

    def translate_file(self, filename, show_tokens=False):
        """Translate a Mochi file into a Python AST.
        """
        self.filename = filename
        with open(filename, 'r') as fobj:
            content = fobj.read()
        sexps = parse(lex(content, debug=show_tokens), filename)
        # TODO: Why this change of directory is needed?
        chdir(normpath(str(Path(filename).parent)))
        return self.translate_sexps(sexps)

    def translate_loaded_file(self, filename, show_tokens=False):
        body = []
        self.filename = filename
        with open(filename, 'r') as f:
            sexps = parse(lex(f.read(), debug=show_tokens), filename)
            for sexp in sexps:
                if isinstance(sexp, MutableSequence):
                    sexp = tuple_it(sexp)
                if sexp is COMMENT:
                    continue
                pre, value = self.translate(sexp)
                body.extend(pre)
                body.append(value)
        return body

    def translate_required_file(self, filename):
        required_filenames = required_filenames_stack[-1]
        if filename in required_filenames:
            return self.translate_ref(NONE_SYM)[1],
        else:
            required_filenames.add(filename)
            return self.translate_loaded_file(filename)

    def translate_loaded_files(self, file_paths):
        body = []
        for file_path in file_paths:
            if not isinstance(file_path, str):
                raise MochiSyntaxError(file_path, self.filename)
            body.extend(self.translate_loaded_file(abspath(file_path)))
        return body, self.translate_ref(NONE_SYM)[1]

    def translate_required_files(self, file_paths):
        body = []
        for file_path in file_paths:
            if not isinstance(file_path, str):
                raise MochiSyntaxError(file_path, self.filename)
            body.extend(self.translate_required_file(abspath(file_path)))
        return body, self.translate_ref(NONE_SYM)[1]

    def translate_sexp_to_interact(self, sexp):
        pre, value = self.translate(sexp, False)
        body = [self.enclose(exp, True) for exp in pre]
        write_value = ast.Call(func=ast.Name(id='write',
                                             ctx=ast.Load(),
                                             lineno=1,
                                             col_offset=0),
                               ctx=ast.Load(),
                               args=[value],
                               keywords=[],
                               starargs=None,
                               kwargs=None,
                               lineno=1,
                               col_offset=0)
        body.append(self.enclose(write_value, True))
        return ast.Module(body=body)

    def translate_sexp(self, sexp):
        # TODO value
        pre, value = self.translate(sexp, False)
        body = [self.enclose(exp, True) for exp in pre]
        return ast.Module(body=body)

    def is_self_evaluating(self, exp):
        return isinstance(exp, (int, float, complex, str, bool, type(None),
                          Keyword))

    def translate_self_evaluating(self, exp):
        if isinstance(exp, (int, float, complex)):
            if hasattr(exp, 'value'):
                return EMPTY, ast.Num(exp.value,
                                      lineno=exp.lineno,
                                      col_offset=0)
            else:
{'fraction_non_alphanumeric': 0.09437413124588485, 'fraction_numerical': 0.0030726461335869484, 'mean_word_length': 1.8815345699831365, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 16, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2611877', 'n_tokens_mistral': 4018, 'n_tokens_neox': 3857, 'n_words': 907}