<commit_before># encoding: utf-8
#
#
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this file,
# You can obtain one at http:# mozilla.org/MPL/2.0/.
#
# Author: <NAME> (<EMAIL>)
#
from __future__ import unicode_literals
from __future__ import division
from pyLibrary import queries

from pyLibrary.collections.matrix import Matrix
from pyLibrary.collections import AND, SUM, OR, UNION
from pyLibrary.dot import nvl, split_field, set_default, Dict, unwraplist
from pyLibrary.dot.lists import DictList
from pyLibrary.dot import listwrap, unwrap
from pyLibrary.queries.qb_usingES_util import aggregates
from pyLibrary.queries import domains, qb_usingES_util
from pyLibrary.queries.filters import simplify_esfilter, TRUE_FILTER
from pyLibrary.debugs.logs import Log
from pyLibrary.queries import MVEL, filters
from pyLibrary.queries.cube import Cube


def is_fieldop(query):
    # THESE SMOOTH EDGES REQUIRE ALL DATA (SETOP)

    select = listwrap(query.select)
    if not query.edges:
        isDeep = len(split_field(query.frum.name)) > 1  # LOOKING INTO NESTED WILL REQUIRE A SCRIPT
        isSimple = AND(s.value != None and (s.value=="*" or isKeyword(s.value)) for s in select)
        noAgg = AND(s.aggregate == "none" for s in select)

        if not isDeep and isSimple and noAgg:
            return True
    else:
        isSmooth = AND((e.domain.type in domains.ALGEBRAIC and e.domain.interval == "none") for e in query.edges)
        if isSmooth:
            return True

    return False

def isKeyword(value):
    if isinstance(value, dict):
        return AND(isKeyword(v) for k, v in value.items())
    if isinstance(value, list):
        return AND(isKeyword(v) for v in value)
    return MVEL.isKeyword(value)


def es_fieldop(es, query):
    es_query = qb_usingES_util.buildFromES(query)
    select = listwrap(query.select)
    es_query.query = {
        "filtered": {
            "query": {
                "match_all": {}
            },
            "filter": filters.simplify_esfilter(query.where)
        }
    }
    es_query.size = nvl(query.limit, queries.query.DEFAULT_LIMIT)
    es_query.fields = DictList()
    source = "fields"
    for s in select.value:
        if s == "*":
            es_query.fields=None
            source = "_source"
        elif s == ".":
            es_query.fields=None
            source = "_source"
        elif isinstance(s, list) and es_query.fields is not None:
            es_query.fields.extend(s)
        elif isinstance(s, dict) and es_query.fields is not None:
            es_query.fields.extend(s.values())
        elif es_query.fields is not None:
            es_query.fields.append(s)
    es_query.sort = [{s.field: "asc" if s.sort >= 0 else "desc"} for s in query.sort]

    data = qb_usingES_util.post(es, es_query, query.limit)
    T = data.hits.hits


    for i, s in enumerate(select.copy()):
        # IF THERE IS A *, THEN INSERT THE EXTRA COLUMNS
        if s.value == "*":
            try:
                column_names = set(query.frum.get_column_names())
            except Exception, e:
                Log.warning("can not get columns", e)
                column_names = UNION(*[[k for k, v in row.items()] for row in T.select(source)])
            column_names -= set(select.name)
            select = select[:i:] + [{"name": n, "value": n} for n in column_names] + select[i + 1::]
            break

    if query.format == "list":
        data = []
        for row in T:
            r = {}
            for s in select:
                if s.value == ".":
                    r[s.name] = row[source]
                else:
                    r[s.name] = unwraplist(row[source][s.value])
            data.append(r)
        return Dict(
            meta={"format": "list"},
            data=data
        )
    elif query.format == "table":
        header = [s.name for s in select]
        map = {s.name: i for i, s in enumerate(select)} # MAP FROM name TO COLUMN INDEX
        data = []
        for row in T:
            r = [None]*len(header)
            for s in select:
                if s.value == ".":
                    r[map[s.name]] = row[source]
                else:
                    r[map[s.name]] = unwraplist(row[source][s.value])
            data.append(r)
        return Dict(
            meta={"format":"table"},
            header=header,
            data=data
        )<commit_msg>default list clause<commit_after># encoding: utf-8
#
#
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this file,
# You can obtain one at http:# mozilla.org/MPL/2.0/.
#
# Author: <NAME> (<EMAIL>)
#
from __future__ import unicode_literals
from __future__ import division
from pyLibrary import queries

from pyLibrary.collections.matrix import Matrix
from pyLibrary.collections import AND, SUM, OR, UNION
from pyLibrary.dot import nvl, split_field, set_default, Dict, unwraplist
from pyLibrary.dot.lists import DictList
from pyLibrary.dot import listwrap, unwrap
from pyLibrary.queries.qb_usingES_util import aggregates
from pyLibrary.queries import domains, qb_usingES_util
from pyLibrary.queries.filters import simplify_esfilter, TRUE_FILTER
from pyLibrary.debugs.logs import Log
from pyLibrary.queries import MVEL, filters
from pyLibrary.queries.cube import Cube


def is_fieldop(query):
    # THESE SMOOTH EDGES REQUIRE ALL DATA (SETOP)

    select = listwrap(query.select)
    if not query.edges:
        isDeep = len(split_field(query.frum.name)) > 1  # LOOKING INTO NESTED WILL REQUIRE A SCRIPT
        isSimple = AND(s.value != None and (s.value=="*" or isKeyword(s.value)) for s in select)
        noAgg = AND(s.aggregate == "none" for s in select)

        if not isDeep and isSimple and noAgg:
            return True
    else:
        isSmooth = AND((e.domain.type in domains.ALGEBRAIC and e.domain.interval == "none") for e in query.edges)
        if isSmooth:
            return True

    return False

def isKeyword(value):
    if isinstance(value, dict):
        return AND(isKeyword(v) for k, v in value.items())
    if isinstance(value, list):
        return AND(isKeyword(v) for v in value)
    return MVEL.isKeyword(value)


def es_fieldop(es, query):
    es_query = qb_usingES_util.buildFromES(query)
    select = listwrap(query.select)
    es_query.query = {
        "filtered": {
            "query": {
                "match_all": {}
            },
            "filter": filters.simplify_esfilter(query.where)
        }
    }
    es_query.size = nvl(query.limit, queries.query.DEFAULT_LIMIT)
    es_query.fields = DictList()
    source = "fields"
    for s in select.value:
        if s == "*":
            es_query.fields=None
            source = "_source"
        elif s == ".":
            es_query.fields=None
            source = "_source"
        elif isinstance(s, list) and es_query.fields is not None:
            es_query.fields.extend(s)
        elif isinstance(s, dict) and es_query.fields is not None:
            es_query.fields.extend(s.values())
        elif es_query.fields is not None:
            es_query.fields.append(s)
    es_query.sort = [{s.field: "asc" if s.sort >= 0 else "desc"} for s in query.sort]

    data = qb_usingES_util.post(es, es_query, query.limit)
    T = data.hits.hits


    for i, s in enumerate(select.copy()):
        # IF THERE IS A *, THEN INSERT THE EXTRA COLUMNS
        if s.value == "*":
            try:
                column_names = set(query.frum.get_column_names())
            except Exception, e:
                Log.warning("can not get columns", e)
                column_names = UNION(*[[k for k, v in row.items()] for row in T.select(source)])
            column_names -= set(select.name)
            select = select[:i:] + [{"name": n, "value": n} for n in column_names] + select[i + 1::]
            break

    if not query.format or query.format == "list":
        data = []
        for row in T:
            r = {}
            for s in select:
                if s.value == ".":
                    r[s.name] = row[source]
                else:
                    r[s.name] = unwraplist(row[source][s.value])
            data.append(r)
        return Dict(
            meta={"format": "list"},
            data=data
        )
    elif query.format == "table":
        header = [s.name for s in select]
        map = {s.name: i for i, s in enumerate(select)} # MAP FROM name TO COLUMN INDEX
        data = []
        for row in T:
            r = [None]*len(header)
            for s in select:
                if s.value == ".":
                    r[map[s.name]] = row[source]
                else:
                    r[map[s.name]] = unwraplist(row[source][s.value])
            data.append(r)
        return Dict(
            meta={"format":"table"},
            header=header,
            data=data
        )
{'fraction_non_alphanumeric': 0.10933660933660934, 'fraction_numerical': 0.0017869108778199687, 'mean_word_length': 2.1061394380853278, 'pattern_counts': {'":': 30, '<': 7, '<?xml version=': 0, '>': 11, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6331123', 'n_tokens_mistral': 2840, 'n_tokens_neox': 2736, 'n_words': 809}