<commit_before># encoding: utf-8
#
#
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this file,
# You can obtain one at http:# mozilla.org/MPL/2.0/.
#
# Author: <NAME> (<EMAIL>)
#
from __future__ import absolute_import
from __future__ import division
from __future__ import unicode_literals

import itertools
from collections import Mapping
from decimal import Decimal

from pyLibrary import convert
from pyLibrary.collections import OR, MAX
from pyLibrary.debugs.logs import Log
from pyLibrary.dot import coalesce, wrap, set_default, literal_field, listwrap, Null
from pyLibrary.queries.domains import is_keyword
from pyLibrary.times.dates import Date

ALLOW_SCRIPTING = False
TRUE_FILTER = True
FALSE_FILTER = False

_Query = None


def _late_import():
    global _Query

    from pyLibrary.queries.query import Query as _Query

    _ = _Query


def jx_expression(expr):
    """
    WRAP A JSON EXPRESSION WITH OBJECT REPRESENTATION
    """
    if expr in (True, False, None) or expr == None or isinstance(expr, (float, int, Decimal)) or isinstance(expr, Date):
        return Literal(None, expr)
    elif is_keyword(expr):
        return Variable(expr)
    elif expr == "":
        Log.error("expression is empty")

    try:
        items = expr.items()
    except Exception, e:
        Log.error("programmer error expr = {{value|quote}}", value=expr, cause=e)
    op, term = items[0]

    if len(items) == 1:
        class_ = operators.get(op)
        if not class_:
            Log.error("{{operator|quote}} is not a known operator", operator=op)
        clauses = {}
    else:
        for item in items:
            op, term = item
            class_ = operators.get(op)
            if class_:
                clauses = {k: jx_expression(v) for k, v in expr.items() if k != op}
                break
        else:
            raise Log.error("{{operator|quote}} is not a known operator", operator=op)

    if class_ is Literal:
        return class_(op, term)
    elif class_ is ScriptOp:
        if ALLOW_SCRIPTING:
            Log.warning("Scripting has been activated:  This has known security holes!!\nscript = {{script||quote}}", script=term)
            return class_(op, term)
        else:
            Log.error("scripting is disabled")
    elif term == None:
        return class_(op, [], **clauses)
    elif isinstance(term, list):
        terms = map(jx_expression, term)
        return class_(op, terms, **clauses)
    elif isinstance(term, Mapping):
        items = term.items()
        if class_.has_simple_form:
            if len(items) == 1:
                k, v = items[0]
                return class_(op, [Variable(k), Literal(None, v)], **clauses)
            else:
                return class_(op, {k: Literal(None, v) for k, v in items}, **clauses)
        else:
            return class_(op, jx_expression(term), **clauses)
    else:
        return class_(op, jx_expression(term), **clauses)


def compile_expression(source):
    # FORCE MODULES TO BE IN NAMESPACE
    _ = coalesce
    _ = Date
    _ = convert

    output = None
    exec """
def output(row, rownum=None, rows=None):
    try:
        return """ + source + """
    except Exception, e:
        Log.error("Problem with dynamic function {{func|quote}}",  func= """ + convert.value2quote(source) + """, cause=e)
"""
    return output


def jx_expression_to_function(expr):
    """
    RETURN FUNCTION THAT REQUIRES PARAMETERS (row, rownum=None, rows=None):
    """
    if expr != None and not isinstance(expr, (Mapping, list)) and hasattr(expr, "__call__"):
        return expr
    return compile_expression(jx_expression(expr).to_python())


def query_get_all_vars(query, exclude_where=False):
    """
    :param query:
    :param exclude_where: Sometimes we do not what to look at the where clause
    :return: all variables in use by query
    """
    output = set()
    for s in listwrap(query.select):
        output |= select_get_all_vars(s)
    for s in listwrap(query.edges):
        output |= edges_get_all_vars(s)
    for s in listwrap(query.groupby):
        output |= edges_get_all_vars(s)
    if not exclude_where:
        output |= jx_expression(query.where).vars()
    return output


def select_get_all_vars(s):
    if isinstance(s.value, list):
        return set(s.value)
    elif isinstance(s.value, basestring):
        return {s.value}
    elif s.value == None or s.value == ".":
        return set()
    else:
        if s.value == "*":
            return {"*"}
        return jx_expression(s.value).vars()


def edges_get_all_vars(e):
    output = set()
    if isinstance(e.value, basestring):
        output.add(e.value)
    if e.domain.key:
        output.add(e.domain.key)
    if e.domain.where:
        output |= jx_expression(e.domain.where).vars()
    if e.domain.partitions:
        for p in e.domain.partitions:
            if p.where:
                output |= jx_expression(p.where).vars()
    return output


class Expression(object):
    has_simple_form = False


    def __init__(self, op, terms):
        if isinstance(terms, (list, tuple)):
            if not all(isinstance(t, Expression) for t in terms):
                Log.error("Expecting an expression")
        elif isinstance(terms, Mapping):
            if not all(isinstance(k, Variable) and isinstance(v, Literal) for k, v in terms.items()):
                Log.error("Expecting an {<variable>: <literal}")
        elif terms == None:
            pass
        else:
            if not isinstance(terms, Expression):
                Log.error("Expecting an expression")

    def to_ruby(self, not_null=False, boolean=False):
        """
        :param not_null:  (Optimization) SET TO True IF YOU KNOW THIS EXPRESSION CAN NOT RETURN null
        :param boolean:   (Optimization) SET TO True IF YOU WANT A BOOLEAN RESULT
        :return: jRuby/ES code (unicode)
        """
        raise NotImplementedError

    def to_python(self, not_null=False, boolean=False):
        """
        :param not_null:  (Optimization) SET TO True IF YOU KNOW THIS EXPRESSION CAN NOT RETURN null
        :param boolean:   (Optimization) SET TO True IF YOU WANT A BOOLEAN RESULT
        :return: Python code (unicode)
        """
        raise Log.error("{{type}} has no `to_python` method", type=self.__class__.__name__)

    def to_esfilter(self):
        raise Log.error("{{type}} has no `to_esfilter` method", type=self.__class__.__name__)

    def to_dict(self):
        raise NotImplementedError

    def __json__(self):
        return convert.value2json(self.to_dict())

    def vars(self):
        raise Log.error("{{type}} has no `vars` method", type=self.__class__.__name__)

    def map(self, map):
        raise Log.error("{{type}} has no `map` method", type=self.__class__.__name__)

    def missing(self):
        # RETURN FILTER THAT INDICATE THIS EXPRESSIOn RETURNS null
        raise Log.error("{{type}} has no `missing` method", type=self.__class__.__name__)

    def exists(self):
        return NotOp("not", self.missing())

    def is_true(self):
        """
        :return: True, IF THIS EXPRESSION ALWAYS RETURNS BOOLEAN true
        """
        return FalseOp()  # GOOD DEFAULT ASSUMPTION

    def is_false(self):
        """
        :return: True, IF THIS EXPRESSION ALWAYS RETURNS BOOLEAN false
        """
        return FalseOp()  # GOOD DEFAULT ASSUMPTION


class Variable(Expression):

    def __init__(self, var):
        Expression.__init__(self, "", None)
        if not is_keyword(var):
            Log.error("Expecting a variable")
        self.var = var

    def to_ruby(self, not_null=False, boolean=False):
        if self.var == ".":
            return "_source"
        else:
            q = convert.string2quote(self.var)
            if not_null:
                if boolean:
                    return "doc[" + q + "].value==\"T\""
                else:
                    return "doc[" + q + "].value"
            else:
                if boolean:
                    return "doc[" + q + "].isEmpty() ? null : (doc[" + q + "].value==\"T\")"
                else:
                    return "doc[" + q + "].isEmpty() ? null : doc[" + q + "].value"

    def to_python(self, not_null=False, boolean=False):
        if self.var == ".":
            return "row"
        else:
            return "row[" + convert.value2quote(self.var) + "]"

    def to_dict(self):
        return self.var

    def vars(self):
        return {self.var}

    def map(self, map_):
        if not isinstance(map_, Mapping):
            Log.error("Expecting Mapping")

        return Variable(coalesce(map_.get(self.var), self.var))

    def missing(self):
        # RETURN FILTER THAT INDICATE THIS EXPRESSION RETURNS null
        return MissingOp("missing", self)

    def exists(self):
        return ExistsOp("exists", self)

    def __hash__(self):
        return self.var.__hash__()

    def __eq__(self, other):
        return self.var.__eq__(other)

    def __unicode__(self):<commit_msg>try this accessor<commit_after># encoding: utf-8
#
#
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this file,
# You can obtain one at http:# mozilla.org/MPL/2.0/.
#
# Author: <NAME> (<EMAIL>)
#
from __future__ import absolute_import
from __future__ import division
from __future__ import unicode_literals

import itertools
from collections import Mapping
from decimal import Decimal

from pyLibrary import convert
from pyLibrary.collections import OR, MAX
from pyLibrary.debugs.logs import Log
from pyLibrary.dot import coalesce, wrap, set_default, literal_field, listwrap, Null, split_field
from pyLibrary.queries.domains import is_keyword
from pyLibrary.times.dates import Date

ALLOW_SCRIPTING = False
TRUE_FILTER = True
FALSE_FILTER = False

_Query = None


def _late_import():
    global _Query

    from pyLibrary.queries.query import Query as _Query

    _ = _Query


def jx_expression(expr):
    """
    WRAP A JSON EXPRESSION WITH OBJECT REPRESENTATION
    """
    if expr in (True, False, None) or expr == None or isinstance(expr, (float, int, Decimal)) or isinstance(expr, Date):
        return Literal(None, expr)
    elif is_keyword(expr):
        return Variable(expr)
    elif expr == "":
        Log.error("expression is empty")

    try:
        items = expr.items()
    except Exception, e:
        Log.error("programmer error expr = {{value|quote}}", value=expr, cause=e)
    op, term = items[0]

    if len(items) == 1:
        class_ = operators.get(op)
        if not class_:
            Log.error("{{operator|quote}} is not a known operator", operator=op)
        clauses = {}
    else:
        for item in items:
            op, term = item
            class_ = operators.get(op)
            if class_:
                clauses = {k: jx_expression(v) for k, v in expr.items() if k != op}
                break
        else:
            raise Log.error("{{operator|quote}} is not a known operator", operator=op)

    if class_ is Literal:
        return class_(op, term)
    elif class_ is ScriptOp:
        if ALLOW_SCRIPTING:
            Log.warning("Scripting has been activated:  This has known security holes!!\nscript = {{script||quote}}", script=term)
            return class_(op, term)
        else:
            Log.error("scripting is disabled")
    elif term == None:
        return class_(op, [], **clauses)
    elif isinstance(term, list):
        terms = map(jx_expression, term)
        return class_(op, terms, **clauses)
    elif isinstance(term, Mapping):
        items = term.items()
        if class_.has_simple_form:
            if len(items) == 1:
                k, v = items[0]
                return class_(op, [Variable(k), Literal(None, v)], **clauses)
            else:
                return class_(op, {k: Literal(None, v) for k, v in items}, **clauses)
        else:
            return class_(op, jx_expression(term), **clauses)
    else:
        return class_(op, jx_expression(term), **clauses)


def compile_expression(source):
    # FORCE MODULES TO BE IN NAMESPACE
    _ = coalesce
    _ = Date
    _ = convert

    output = None
    exec """
def output(row, rownum=None, rows=None):
    try:
        return """ + source + """
    except Exception, e:
        Log.error("Problem with dynamic function {{func|quote}}",  func= """ + convert.value2quote(source) + """, cause=e)
"""
    return output


def jx_expression_to_function(expr):
    """
    RETURN FUNCTION THAT REQUIRES PARAMETERS (row, rownum=None, rows=None):
    """
    if expr != None and not isinstance(expr, (Mapping, list)) and hasattr(expr, "__call__"):
        return expr
    return compile_expression(jx_expression(expr).to_python())


def query_get_all_vars(query, exclude_where=False):
    """
    :param query:
    :param exclude_where: Sometimes we do not what to look at the where clause
    :return: all variables in use by query
    """
    output = set()
    for s in listwrap(query.select):
        output |= select_get_all_vars(s)
    for s in listwrap(query.edges):
        output |= edges_get_all_vars(s)
    for s in listwrap(query.groupby):
        output |= edges_get_all_vars(s)
    if not exclude_where:
        output |= jx_expression(query.where).vars()
    return output


def select_get_all_vars(s):
    if isinstance(s.value, list):
        return set(s.value)
    elif isinstance(s.value, basestring):
        return {s.value}
    elif s.value == None or s.value == ".":
        return set()
    else:
        if s.value == "*":
            return {"*"}
        return jx_expression(s.value).vars()


def edges_get_all_vars(e):
    output = set()
    if isinstance(e.value, basestring):
        output.add(e.value)
    if e.domain.key:
        output.add(e.domain.key)
    if e.domain.where:
        output |= jx_expression(e.domain.where).vars()
    if e.domain.partitions:
        for p in e.domain.partitions:
            if p.where:
                output |= jx_expression(p.where).vars()
    return output


class Expression(object):
    has_simple_form = False


    def __init__(self, op, terms):
        if isinstance(terms, (list, tuple)):
            if not all(isinstance(t, Expression) for t in terms):
                Log.error("Expecting an expression")
        elif isinstance(terms, Mapping):
            if not all(isinstance(k, Variable) and isinstance(v, Literal) for k, v in terms.items()):
                Log.error("Expecting an {<variable>: <literal}")
        elif terms == None:
            pass
        else:
            if not isinstance(terms, Expression):
                Log.error("Expecting an expression")

    def to_ruby(self, not_null=False, boolean=False):
        """
        :param not_null:  (Optimization) SET TO True IF YOU KNOW THIS EXPRESSION CAN NOT RETURN null
        :param boolean:   (Optimization) SET TO True IF YOU WANT A BOOLEAN RESULT
        :return: jRuby/ES code (unicode)
        """
        raise NotImplementedError

    def to_python(self, not_null=False, boolean=False):
        """
        :param not_null:  (Optimization) SET TO True IF YOU KNOW THIS EXPRESSION CAN NOT RETURN null
        :param boolean:   (Optimization) SET TO True IF YOU WANT A BOOLEAN RESULT
        :return: Python code (unicode)
        """
        raise Log.error("{{type}} has no `to_python` method", type=self.__class__.__name__)

    def to_esfilter(self):
        raise Log.error("{{type}} has no `to_esfilter` method", type=self.__class__.__name__)

    def to_dict(self):
        raise NotImplementedError

    def __json__(self):
        return convert.value2json(self.to_dict())

    def vars(self):
        raise Log.error("{{type}} has no `vars` method", type=self.__class__.__name__)

    def map(self, map):
        raise Log.error("{{type}} has no `map` method", type=self.__class__.__name__)

    def missing(self):
        # RETURN FILTER THAT INDICATE THIS EXPRESSIOn RETURNS null
        raise Log.error("{{type}} has no `missing` method", type=self.__class__.__name__)

    def exists(self):
        return NotOp("not", self.missing())

    def is_true(self):
        """
        :return: True, IF THIS EXPRESSION ALWAYS RETURNS BOOLEAN true
        """
        return FalseOp()  # GOOD DEFAULT ASSUMPTION

    def is_false(self):
        """
        :return: True, IF THIS EXPRESSION ALWAYS RETURNS BOOLEAN false
        """
        return FalseOp()  # GOOD DEFAULT ASSUMPTION


class Variable(Expression):

    def __init__(self, var):
        Expression.__init__(self, "", None)
        if not is_keyword(var):
            Log.error("Expecting a variable")
        self.var = var

    def to_ruby(self, not_null=False, boolean=False):
        if self.var == ".":
            return "_source"
        else:
            q = convert.string2quote(self.var)
            if not_null:
                if boolean:
                    return "doc[" + q + "].value==\"T\""
                else:
                    return "doc[" + q + "].value"
            else:
                if boolean:
                    return "doc[" + q + "].isEmpty() ? null : (doc[" + q + "].value==\"T\")"
                else:
                    return "doc[" + q + "].isEmpty() ? null : doc[" + q + "].value"

    def to_python(self, not_null=False, boolean=False):
        path = split_field(self.var)
        agg = "row"
        if not path:
            return agg
        for p in path[:-1]:
            agg = agg+".get("+convert.value2quote(p)+", EMPTY_DICT)"
        return agg+".get("+convert.value2quote(path[-1])+")"

    def __call__(self, row, rownum=None, rows=None):
        path = split_field(self.var)
        for p in path:
            row = row.get(p)
            if row is None:
                return None
        return row

    def to_dict(self):
        return self.var

    def vars(self):
        return {self.var}

    def map(self, map_):
        if not isinstance(map_, Mapping):
            Log.error("Expecting Mapping")

        return Variable(coalesce(map_.get(self.var), self.var))

    def missing(self):
        # RETURN FILTER THAT INDICATE THIS EXPRESSION RETURNS null
        return MissingOp("missing", self)

    def exists(self):
        return ExistsOp("exists", self)

    def __hash__(self):
        return self.var.__hash__()

    def __eq__(self, other):
        return self.var.__eq__(other)

    def __unicode__(self):
{'fraction_non_alphanumeric': 0.11460528448695088, 'fraction_numerical': 0.0015669746582374236, 'mean_word_length': 2.363256405596947, 'pattern_counts': {'":': 9, '<': 11, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '646916', 'n_tokens_mistral': 5682, 'n_tokens_neox': 5467, 'n_words': 1649}