<commit_before>##
# Copyright (c) 2010-2013 Sprymix Inc.
# All rights reserved.
#
# See LICENSE for details.
##


import collections

from metamagic import caos
from metamagic.caos import proto
from metamagic.utils import ast
from . import ast as caosql_ast, parser, transformer, codegen
from metamagic.caos.tree import ast as caos_ast
from metamagic.caos.tree import transformer as caos_transformer
from metamagic.caos import caosql

from metamagic.utils import debug

from . import errors


class CaosQLExpression:
    def __init__(self, proto_schema, module_aliases=None):
        self.parser = parser.CaosQLParser()
        self.module_aliases = module_aliases
        self.proto_schema = proto_schema
        self.transformer = transformer.CaosqlTreeTransformer(proto_schema, module_aliases)
        self.reverse_transformer = transformer.CaosqlReverseTransformer()
        self.path_resolver = None

    def parse_expr(self, expr):
        return self.parser.parse(expr)

    def process_concept_expr(self, expr, concept):
        tree = self.parser.parse(expr)
        context = transformer.ParseContext()
        context.current.location = 'selector'
        return self.transformer._process_expr(context, tree)

    def normalize_refs(self, expr, module_aliases=None):
        tree = self.parser.parse(expr)
        tree = self.transformer.normalize_refs(tree, module_aliases=module_aliases)
        return codegen.CaosQLSourceGenerator.to_source(tree)

    def normalize_tree(self, tree, module_aliases=None, anchors=None, inline_anchors=False):
        if not isinstance(tree, caosql_ast.StatementNode):
            selnode = caosql_ast.SelectQueryNode()
            selnode.targets = [caosql_ast.SelectExprNode(expr=tree)]
            tree = selnode

        caos_tree = self.transformer.transform(tree, (), module_aliases=module_aliases,
                                                         anchors=anchors)
        caosql_tree = self.reverse_transformer.transform(caos_tree, inline_anchors=inline_anchors)
        return caosql_tree, caos_tree

    def normalize_expr(self, expr, module_aliases=None, anchors=None):
        tree = self.parser.parse(expr)
        tree, caos_tree = self.normalize_tree(tree, module_aliases=module_aliases, anchors=anchors)
        return codegen.CaosQLSourceGenerator.to_source(tree), caos_tree

    def transform_expr_fragment(self, expr, anchors=None, location=None):
        tree = self.parser.parse(expr)
        return self.transformer.transform_fragment(tree, (), anchors=anchors, location=location)

    @debug.debug
    def transform_expr(self, expr, anchors=None, arg_types=None,
                                   security_context=None):
        """LOG [caos.query] CaosQL query:
        print(expr)
        """

        caosql_tree = self.parser.parse(expr)

        """LOG [caos.query] CaosQL tree:
        from metamagic.utils import markup
        markup.dump(caosql_tree)
        """

        if not isinstance(caosql_tree, caosql.ast.StatementNode):
            selnode = caosql.ast.SelectQueryNode()
            selnode.targets = [caosql.ast.SelectExprNode(expr=caosql_tree)]
            caosql_tree = selnode

        query_tree = self.transformer.transform(
                        caosql_tree, arg_types,
                        module_aliases=self.module_aliases,
                        anchors=anchors, security_context=security_context)

        """LOG [caos.query] Caos tree:
        from metamagic.utils import markup
        markup.dump(query_tree)
        """

        return query_tree

    def get_path_targets(self, expr, session, anchors=None):
        if not self.path_resolver:
            self.path_resolver = caos_transformer.PathResolver()
        ctree = self.transform_expr_fragment(expr, anchors=anchors)
        targets = self.path_resolver.resolve_path(ctree, session)
        return targets

    def normalize_source_expr(self, expr, source):
        tree = self.parser.parse(expr)

        visitor = _PrependSource(source, self.proto_schema, self.module_aliases)
        visitor.visit(tree)

        expr = codegen.CaosQLSourceGenerator.to_source(tree)
        return expr, tree

    def check_source_atomic_expr(self, tree, source):
        context = transformer.ParseContext()
        context.current.location = 'selector'
        processed = self.transformer._process_expr(context, tree)

        ok = isinstance(processed, caos_ast.BaseRef) \
             or (isinstance(processed, caos_ast.Disjunction) and
                 isinstance(list(processed.paths)[0], caos_ast.BaseRef))

        if not ok:
            msg = "invalid link reference"
            details = "Expression must only contain references to local atoms"
            raise errors.CaosQLReferenceError(msg, details=details)

        return processed

    def get_source_references(self, tree):
        result = []

        refs = self.transformer.extract_paths(tree, reverse=True, resolve_arefs=True,
                                                    recurse_subqueries=-1)

        if refs is not None:
            flt = lambda n: isinstance(n, (caos_ast.EntitySet, caos_ast.EntityLink))
            nodes = ast.find_children(refs, flt)<commit_msg>caos.caosql: Omit formatting in various normalize* helpers<commit_after>##
# Copyright (c) 2010-2013 Sprymix Inc.
# All rights reserved.
#
# See LICENSE for details.
##


import collections

from metamagic import caos
from metamagic.caos import proto
from metamagic.utils import ast
from . import ast as caosql_ast, parser, transformer, codegen
from metamagic.caos.tree import ast as caos_ast
from metamagic.caos.tree import transformer as caos_transformer
from metamagic.caos import caosql

from metamagic.utils import debug

from . import errors


class CaosQLExpression:
    def __init__(self, proto_schema, module_aliases=None):
        self.parser = parser.CaosQLParser()
        self.module_aliases = module_aliases
        self.proto_schema = proto_schema
        self.transformer = transformer.CaosqlTreeTransformer(proto_schema, module_aliases)
        self.reverse_transformer = transformer.CaosqlReverseTransformer()
        self.path_resolver = None

    def parse_expr(self, expr):
        return self.parser.parse(expr)

    def process_concept_expr(self, expr, concept):
        tree = self.parser.parse(expr)
        context = transformer.ParseContext()
        context.current.location = 'selector'
        return self.transformer._process_expr(context, tree)

    def normalize_refs(self, expr, module_aliases=None):
        tree = self.parser.parse(expr)
        tree = self.transformer.normalize_refs(tree, module_aliases=module_aliases)
        return codegen.CaosQLSourceGenerator.to_source(tree, pretty=False)

    def normalize_tree(self, tree, module_aliases=None, anchors=None, inline_anchors=False):
        if not isinstance(tree, caosql_ast.StatementNode):
            selnode = caosql_ast.SelectQueryNode()
            selnode.targets = [caosql_ast.SelectExprNode(expr=tree)]
            tree = selnode

        caos_tree = self.transformer.transform(tree, (), module_aliases=module_aliases,
                                                         anchors=anchors)
        caosql_tree = self.reverse_transformer.transform(caos_tree, inline_anchors=inline_anchors)
        return caosql_tree, caos_tree

    def normalize_expr(self, expr, module_aliases=None, anchors=None):
        tree = self.parser.parse(expr)
        tree, caos_tree = self.normalize_tree(tree, module_aliases=module_aliases, anchors=anchors)
        return codegen.CaosQLSourceGenerator.to_source(tree, pretty=False), caos_tree

    def transform_expr_fragment(self, expr, anchors=None, location=None):
        tree = self.parser.parse(expr)
        return self.transformer.transform_fragment(tree, (), anchors=anchors, location=location)

    @debug.debug
    def transform_expr(self, expr, anchors=None, arg_types=None,
                                   security_context=None):
        """LOG [caos.query] CaosQL query:
        print(expr)
        """

        caosql_tree = self.parser.parse(expr)

        """LOG [caos.query] CaosQL tree:
        from metamagic.utils import markup
        markup.dump(caosql_tree)
        """

        if not isinstance(caosql_tree, caosql.ast.StatementNode):
            selnode = caosql.ast.SelectQueryNode()
            selnode.targets = [caosql.ast.SelectExprNode(expr=caosql_tree)]
            caosql_tree = selnode

        query_tree = self.transformer.transform(
                        caosql_tree, arg_types,
                        module_aliases=self.module_aliases,
                        anchors=anchors, security_context=security_context)

        """LOG [caos.query] Caos tree:
        from metamagic.utils import markup
        markup.dump(query_tree)
        """

        return query_tree

    def get_path_targets(self, expr, session, anchors=None):
        if not self.path_resolver:
            self.path_resolver = caos_transformer.PathResolver()
        ctree = self.transform_expr_fragment(expr, anchors=anchors)
        targets = self.path_resolver.resolve_path(ctree, session)
        return targets

    def normalize_source_expr(self, expr, source):
        tree = self.parser.parse(expr)

        visitor = _PrependSource(source, self.proto_schema, self.module_aliases)
        visitor.visit(tree)

        expr = codegen.CaosQLSourceGenerator.to_source(tree, pretty=False)
        return expr, tree

    def check_source_atomic_expr(self, tree, source):
        context = transformer.ParseContext()
        context.current.location = 'selector'
        processed = self.transformer._process_expr(context, tree)

        ok = isinstance(processed, caos_ast.BaseRef) \
             or (isinstance(processed, caos_ast.Disjunction) and
                 isinstance(list(processed.paths)[0], caos_ast.BaseRef))

        if not ok:
            msg = "invalid link reference"
            details = "Expression must only contain references to local atoms"
            raise errors.CaosQLReferenceError(msg, details=details)

        return processed

    def get_source_references(self, tree):
        result = []

        refs = self.transformer.extract_paths(tree, reverse=True, resolve_arefs=True,
                                                    recurse_subqueries=-1)

        if refs is not None:
            flt = lambda n: isinstance(n, (caos_ast.EntitySet, caos_ast.EntityLink))
            nodes = ast.find_children(refs, flt)
{'fraction_non_alphanumeric': 0.08508413347276357, 'fraction_numerical': 0.0019013214183857781, 'mean_word_length': 2.77737881508079, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 10, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '783868', 'n_tokens_mistral': 3158, 'n_tokens_neox': 2962, 'n_words': 622}