<commit_before># -*- coding utf-8 -*-
"""
Will accept a tree of allowed symbols, and construct an abstract syntax tree
"""
from __future__ import unicode_literals, absolute_import

from whispy_lispy import ast, cst


def get_ast(tree):
    """

    :param tree: A Syntax tree(?) - anyway a possibly nested list produced
        by the lexer
    :return: an abstract syntax tree
    """
    abstract_syntax_tree = []
    if not tree:
        return abstract_syntax_tree

    for elem in tree:
        for operation in (ast.Assign, ast.Quote, ast.Eval, ast.Symbol, ast.Literal, ast.Apply):  # noqa
            match = operation.matches(elem)
            if match is not None:
                abstract_syntax_tree.append(operation.from_match(match))
                break

    return abstract_syntax_tree


def get_ast2(cstree):
    """Convert the concrete syntax tree into an abstract one

    :type cstree: cst.ConcreteSyntaxNode
    :rtype: ast.AbstractSyntaxNode
    """

    result = transform_one_to_one(cstree)
    return mutate_tree_structure(result)

def mutate_tree_structure(tree):
    """Apply transformations to the abstract syntax tree that alter its
    structure

    I wonder if this is really necessary... I mean just replace the quote
    with an Apply('quote', X), and we can keep the structure, and just mutate
    one element "'" -> "quote"...we'll see

    :type tree: ast.AbstractSyntaxNode
    :rtype: ast.AbstractSyntaxNode
    """
    # TODO - transform >>> ' a b ...<<< into >>> (quote a) b ... <<< - no hurry
    # if not tree.is_leaf():
    #     new_children = []
    #     if isinstance(tree, ast.Quote2):
    #         iterable_values = tree.values[1:]
    #     else:
    #         iterable_values = tree.values
    #
    #     for child in iterable_values:
    #         new_children.append(mutate_tree_structure(child))
    #
    #     return tree.alike(tuple(new_children))
    # if isinstance(tree, ast.Apply2):
    #     if
    if not tree.is_leaf():
        new_children = []
        skip_next = False
        for idx, child in enumerate(tree.values):
            if skip_next:
                skip_next = False
                continue
            # Might be the case that the quote is the last element
            # Must throw some kind of error or something
            if isinstance(child, ast.OperatorQuote):
                skip_next = True
                new_children.append(
                    mutate_tree_structure(
                        ast.Apply2((ast.Quote2((tree.values[idx + 1], )),))))

    return tree


def transform_one_to_one(cstree):
    """Transform a concrete syntax tree into an abstract one, preserving the
    tree structure

    :type cstree: cst.ConcreteSyntaxNode
    :rtype: ast.AbstractSyntaxNode
    """
    values = []
    if cstree.is_leaf():
        node_class = determine_operation_type(cstree)
        return node_class(tuple(cstree.values))

    for value in cstree.values:
        if value.is_leaf():
            node_class = determine_operation_type(value)
            values.append(node_class(tuple(value.values)))
        else:
            values.append(transform_one_to_one(value))
    return determine_operation_type(cstree)(tuple(values))


def determine_operation_type(cstree):
    """Determine the operation type that this node corresponds to<commit_msg>cleaned up the parser<commit_after># -*- coding utf-8 -*-
"""
Will accept a tree of allowed symbols, and construct an abstract syntax tree
"""
from __future__ import unicode_literals, absolute_import

from whispy_lispy import ast, cst


def get_ast(tree):
    """

    :param tree: A Syntax tree(?) - anyway a possibly nested list produced
        by the lexer
    :return: an abstract syntax tree
    """
    abstract_syntax_tree = []
    if not tree:
        return abstract_syntax_tree

    for elem in tree:
        for operation in (ast.Assign, ast.Quote, ast.Eval, ast.Symbol, ast.Literal, ast.Apply):  # noqa
            match = operation.matches(elem)
            if match is not None:
                abstract_syntax_tree.append(operation.from_match(match))
                break

    return abstract_syntax_tree


def get_ast2(cstree):
    """Convert the concrete syntax tree into an abstract one

    :type cstree: cst.ConcreteSyntaxNode
    :rtype: ast.AbstractSyntaxNode
    """

    result = transform_one_to_one(cstree)
    return mutate_tree_structure(result)


def mutate_tree_structure(tree):
    """Apply transformations to the abstract syntax tree that alter its
    structure

    :type tree: ast.AbstractSyntaxNode
    :rtype: ast.AbstractSyntaxNode
    """
    result = transform_quote_operator_into_function(tree)
    return result

def transform_quote_operator_into_function(tree):
    "Transform  >' a b ...< into  >(quote a) b ...< """
    if not tree.is_leaf():
        new_children = []
        skip_next = False
        for idx, child in enumerate(tree.values):
            if skip_next:
                skip_next = False
                continue
            # Might be the case that the quote is the last element
            # Must throw some kind of error or something
            if isinstance(child, ast.OperatorQuote):
                skip_next = True
                new_children.append(
                    mutate_tree_structure(
                        ast.Apply2((ast.Quote2((tree.values[idx + 1], )),))))

    return tree



def transform_one_to_one(cstree):
    """Transform a concrete syntax tree into an abstract one, preserving the
    tree structure

    :type cstree: cst.ConcreteSyntaxNode
    :rtype: ast.AbstractSyntaxNode
    """
    values = []
    if cstree.is_leaf():
        node_class = determine_operation_type(cstree)
        return node_class(tuple(cstree.values))

    for value in cstree.values:
        if value.is_leaf():
            node_class = determine_operation_type(value)
            values.append(node_class(tuple(value.values)))
        else:
            values.append(transform_one_to_one(value))
    return determine_operation_type(cstree)(tuple(values))


def determine_operation_type(cstree):
    """Determine the operation type that this node corresponds to
{'fraction_non_alphanumeric': 0.08794946550048591, 'fraction_numerical': 0.002105604146420473, 'mean_word_length': 2.490672696438666, 'pattern_counts': {'":': 0, '<': 11, '<?xml version=': 0, '>': 12, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7290096', 'n_tokens_mistral': 1793, 'n_tokens_neox': 1711, 'n_words': 504}