<commit_before>from normality import ascii_text

from aleph.search.util import add_filter


def match_all():
    return {'match_all': {}}


def text_query_string(text, literal=False):
    if text is None or not len(text.strip()):
        return match_all()
    if literal:
        text = '"%s"' % ascii_text(text)
    return {
        'query_string': {
            'query': text,
            'fields': ['text'],
            'default_operator': 'AND',
            'use_dis_max': True
        }
    }


def authz_filter(q, authz, roles=False):
    if authz.is_admin:
        return q

    fq = {'terms': {'collection_id': list(authz.collections_read)}}
    if roles:
        fq = {
            "or": [{'terms': {'roles': list(authz.roles)}}, fq]
        }

    return add_filter(q, fq)


def text_query(text):
    """Part of a query which finds a piece of text."""
    if text is None or not len(text.strip()):
        return match_all()
    return {
        "query_string": {
            "query": text,
            "fields": ['title^5', 'title_latin^4',
                       'summary^3', 'summary_latin^2',
                       'file_name', 'text', '_all'],
            "default_operator": "AND",
            "use_dis_max": True
        }
    }


def multi_match(text, fields, fuzziness=0):
    q = {
        'multi_match': {<commit_msg>use bool not or for authz query<commit_after>from normality import ascii_text

from aleph.search.util import add_filter


def match_all():
    return {'match_all': {}}


def text_query_string(text, literal=False):
    if text is None or not len(text.strip()):
        return match_all()
    if literal:
        text = '"%s"' % ascii_text(text)
    return {
        'query_string': {
            'query': text,
            'fields': ['text'],
            'default_operator': 'AND',
            'use_dis_max': True
        }
    }


def authz_filter(q, authz, roles=False):
    if authz.is_admin:
        return q

    fq = {'terms': {'collection_id': list(authz.collections_read)}}

    if roles:
        iq = {'terms': {'roles': list(authz.roles)}}
        fq = {'bool': {
            'should': [iq, fq],
            'minimum_should_match': 1
        }}
    return add_filter(q, fq)


def text_query(text):
    """Part of a query which finds a piece of text."""
    if text is None or not len(text.strip()):
        return match_all()
    return {
        "query_string": {
            "query": text,
            "fields": ['title^5', 'title_latin^4',
                       'summary^3', 'summary_latin^2',
                       'file_name', 'text', '_all'],
            "default_operator": "AND",
            "use_dis_max": True
        }
    }


def multi_match(text, fields, fuzziness=0):
    q = {
        'multi_match': {
{'fraction_non_alphanumeric': 0.14383809179616913, 'fraction_numerical': 0.003975424647632815, 'mean_word_length': 2.025136612021858, 'pattern_counts': {'":': 11, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7188055', 'n_tokens_mistral': 913, 'n_tokens_neox': 889, 'n_words': 221}