<commit_before>def add_getter(getters, getter_name, field_name, additional_level=None):
    if getter_name is None:
        return
    getter = None
    if additional_level is None:
        def getter_(response_body):
            return response_body[field_name]
        getter = getter_
    else:
        def getter_(response_body):
            return response_body[additional_level][field_name]
        getter = getter_
        pass

    if getter_name is not None:
        getters.update({
            getter_name: getter
        })
        pass
    pass


def agg(body):
    aggs_bodys = {key: body["aggs"][key]["body"] for key in body["aggs"]}
    getters = {getter: body["getter_updater"](body["aggs"][aggr]["getters"][getter], aggr) for aggr in body["aggs"] for getter in body["aggs"][aggr]["getters"] if getter != "self"}
    getters.update(body["getters"])

    body = {**body["body"], **({"aggs": aggs_bodys} if len(aggs_bodys) > 0 else {})}
    return {"body": body, "getters": getters}


def bucket_agg(func):
    def decorated_agg(*args, **kwargs):
        return agg(func(*args, **kwargs))
    return decorated_agg


def request(query=None, fieldlist=None, sorting=None, **aggs):
    """
    Creates core request body. \n
    :param query: Pure JSON query body. Empty by default. \n
    :param fieldlist: List of fields to write in "_source". Empty by default, which means that all source fields will be included in response hits \n
    :param sorting: Sorting dict, {"filed1": "asc", "field2": "desc} for example \n
    :param aggs: Aggregation objects provided by agg functions \n
    :return: {"body": %plain json request%, "getters": %dictionary of named getters%} \n
    """
    if query is None:
        query = {}
    aggs_bodys = {key: aggs[key]["body"] for key in aggs}
    body = {
        **(
            {
                "query": query
            } if len(query) > 0 else {}
        ),
        **(
            {
                "aggs": aggs_bodys
            } if len(aggs_bodys) > 0 else {}
        ),
        **(
            {
                "_source": fieldlist
            } if fieldlist is not None else {}
        ),
        **(
            {
                "sort":
                {
                    skey:
                    {
                        "order": sorting[skey]
                    } for skey in sorting
                }
            } if sorting is not None else {}
        )
    }

    def getter_updater(p_getter, key):
        def deeper_getter(response_body):
            return p_getter(response_body["aggregations"][key])
        return deeper_getter
    getters = {key: getter_updater(aggs[aggr]["getters"][key], aggr) for aggr in aggs for key in aggs[aggr]["getters"]}
    return {"body": body, "getters": getters}


##############################################################################################
# QUERIES                                                                                    #
##############################################################################################


def query_filter(flt):
    return {
        "filtered":
        {
            "filter": flt
        }
    }


##############################################################################################
# FILTERS                                                                                    #
##############################################################################################


def flt_and(*args):
    return {
        "bool":
        {
            "must": args
        }
    }


def flt_exc(*args):
    return {
        "bool": {
            "must_not": args
        }
    }


def flt_or(*args):
    return {
        "bool": {
            "should": args
        }
    }


def flt_eq(field, value):
    if isinstance(value, list):
        return {"terms": {field: value}}
    elif isinstance(value, str) and value[-1] == '*':
        return {"prefix": {field: value[:-1]}}
    return {"term": {field: value}}


def flt_exists(field):
    return {"exists": {"field": field}}


def flt_range(field, left=None, right=None, left_is_strict=False, right_is_strict=False):
    return {"range": {field: {**({"gt" + ("" if left_is_strict else "e"): left} if left is not None else {}), **({ "lt" + ("" if right_is_strict else "e"): right} if right is not None else {})}}}


##############################################################################################
# BUCKET AGGS                                                                                #
##############################################################################################


@bucket_agg
def agg_filter(flt, getter_name=None, **kwargs):
    getters = {}
    add_getter(getters, getter_name, "value")

    def getter_updater(getter, key):
        def deeper_getter(response_body):
            return getter(response_body[key])
        return deeper_getter

    body = {"filter": flt}

    return {"body": body, "getters": getters, "getter_updater": getter_updater, "aggs": kwargs}


@bucket_agg
def agg_terms(field, script=False, size=10000, min_doc_count=None, order=None, getter_doc_count=None, getter_key=None, **kwargs):
    getters = {}
    add_getter(getters, getter_doc_count, "doc_count")
    add_getter(getters, getter_key, "key")

    def getter_factory(key):
        def result(response_body, *args, **kwargs2):
            return [getters[key](bucket) for bucket in response_body["buckets"]]
        return result

    for getter in getters:
        getters[getter] = getter_factory(getter)

    def getter_updater(getter, key):
        def deeper_getter(response_body):
            return [getter(bucket[key]) for bucket in getter(response_body["buckets"])]
        return deeper_getter

    body = {"terms": {
        **{"script" if script else "field": field, "size": size},
        **({"min_doc_count": min_doc_count} if min_doc_count is not None else {}),
        **({"order": order} if order is not None else {})}
    }<commit_msg>kinda axises in terms agg<commit_after>def add_getter(getters, getter_name, field_name, additional_level=None):
    if getter_name is None:
        return
    getter = None
    if additional_level is None:
        def getter_(response_body):
            return response_body[field_name]
        getter = getter_
    else:
        def getter_(response_body):
            return response_body[additional_level][field_name]
        getter = getter_
        pass

    if getter_name is not None:
        getters.update({
            getter_name: getter
        })
        pass
    pass


def agg(body):
    aggs_bodys = {key: body["aggs"][key]["body"] for key in body["aggs"]}
    getters = {getter: body["getter_updater"](body["aggs"][aggr]["getters"][getter], aggr) for aggr in body["aggs"] for getter in body["aggs"][aggr]["getters"] if getter != "self"}
    getters.update(body["getters"])

    body = {**body["body"], **({"aggs": aggs_bodys} if len(aggs_bodys) > 0 else {})}
    return {"body": body, "getters": getters}


def bucket_agg(func):
    def decorated_agg(*args, **kwargs):
        return agg(func(*args, **kwargs))
    return decorated_agg


def request(query=None, fieldlist=None, sorting=None, **aggs):
    """
    Creates core request body. \n
    :param query: Pure JSON query body. Empty by default. \n
    :param fieldlist: List of fields to write in "_source". Empty by default, which means that all source fields will be included in response hits \n
    :param sorting: Sorting dict, {"filed1": "asc", "field2": "desc} for example \n
    :param aggs: Aggregation objects provided by agg functions \n
    :return: {"body": %plain json request%, "getters": %dictionary of named getters%} \n
    """
    if query is None:
        query = {}
    aggs_bodys = {key: aggs[key]["body"] for key in aggs}
    body = {
        **(
            {
                "query": query
            } if len(query) > 0 else {}
        ),
        **(
            {
                "aggs": aggs_bodys
            } if len(aggs_bodys) > 0 else {}
        ),
        **(
            {
                "_source": fieldlist
            } if fieldlist is not None else {}
        ),
        **(
            {
                "sort":
                {
                    skey:
                    {
                        "order": sorting[skey]
                    } for skey in sorting
                }
            } if sorting is not None else {}
        )
    }

    def getter_updater(p_getter, key):
        def deeper_getter(response_body):
            return p_getter(response_body["aggregations"][key])
        return deeper_getter
    getters = {key: getter_updater(aggs[aggr]["getters"][key], aggr) for aggr in aggs for key in aggs[aggr]["getters"]}
    return {"body": body, "getters": getters}


##############################################################################################
# QUERIES                                                                                    #
##############################################################################################


def query_filter(flt):
    return {
        "filtered":
        {
            "filter": flt
        }
    }


##############################################################################################
# FILTERS                                                                                    #
##############################################################################################


def flt_and(*args):
    return {
        "bool":
        {
            "must": args
        }
    }


def flt_exc(*args):
    return {
        "bool": {
            "must_not": args
        }
    }


def flt_or(*args):
    return {
        "bool": {
            "should": args
        }
    }


def flt_eq(field, value):
    if isinstance(value, list):
        return {"terms": {field: value}}
    elif isinstance(value, str) and value[-1] == '*':
        return {"prefix": {field: value[:-1]}}
    return {"term": {field: value}}


def flt_exists(field):
    return {"exists": {"field": field}}


def flt_range(field, left=None, right=None, left_is_strict=False, right_is_strict=False):
    return {"range": {field: {**({"gt" + ("" if left_is_strict else "e"): left} if left is not None else {}), **({ "lt" + ("" if right_is_strict else "e"): right} if right is not None else {})}}}


##############################################################################################
# BUCKET AGGS                                                                                #
##############################################################################################


@bucket_agg
def agg_filter(flt, getter_name=None, **kwargs):
    getters = {}
    add_getter(getters, getter_name, "value")

    def getter_updater(getter, key):
        def deeper_getter(response_body):
            return getter(response_body[key])
        return deeper_getter

    body = {"filter": flt}

    return {"body": body, "getters": getters, "getter_updater": getter_updater, "aggs": kwargs}


@bucket_agg
def agg_terms(field, script=False, size=10000, min_doc_count=None, order=None, getter_doc_count=None, getter_key=None, is_axis=True, **kwargs):
    getters = {}
    add_getter(getters, getter_doc_count, "doc_count")
    add_getter(getters, getter_key, "key")

    def getter_factory(key, bucket_id=None):
        def result(response_body, *args, **kwargs2):
            return [getters[key](bucket) for bucket in response_body["buckets"]]

        def result2(response_body, bucket_id, *args, **kwargs2):
            return getters[key](response_body["buckets"][bucket_id])

        if is_axis:
            return result
        else:
            return result2

    for getter in getters:
        getters[getter] = getter_factory(getter)

    def getter_updater(getter, key):
        def deeper_getter(response_body):
            return [getter(bucket[key]) for bucket in getter(response_body["buckets"])]
        return deeper_getter

    body = {"terms": {
        **{"script" if script else "field": field, "size": size},
        **({"min_doc_count": min_doc_count} if min_doc_count is not None else {}),
        **({"order": order} if order is not None else {})}
    }
{'fraction_non_alphanumeric': 0.2087445958071621, 'fraction_numerical': 0.00236560894037034, 'mean_word_length': 2.3524747060432047, 'pattern_counts': {'":': 76, '<': 3, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 18, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4530351', 'n_tokens_mistral': 3630, 'n_tokens_neox': 3466, 'n_words': 884}