<commit_before>from aleph.model import Entity
from aleph.index import TYPE_RECORD
from aleph.core import es, es_index, url_for


def records_query(document_id, args, size=5, snippet_size=50):
    terms = []
    text = args.get('q', '').strip()
    if len(text):
        terms.append(text)

    entities = Entity.by_id_set(args.getlist('entity'))
    for entity in entities.values():
        terms.extend(entity.terms)

    if not len(terms):
        return None

    shoulds = []
    for term in terms:
        shoulds.append({
            'query_string': {
                'query': text,
                'fields': ['text^10', 'text_latin'],
                'default_operator': 'AND',
                'use_dis_max': True
            }
        })

    q = {
        'bool': {
            'minimum_should_match': 1,
            'should': shoulds
        }
    }
    if document_id is not None:
        q['bool']['must'] = {
            'term': {'document_id': document_id}
        }

    try:
        snippet = int(args.get('snippet', snippet_size))
    except:
        snippet = snippet_size

    return {
        'size': size,
        'query': q,
        'highlight': {
            'fields': {<commit_msg>Fix up records queries. <commit_after>from aleph.model import Entity
from aleph.index import TYPE_RECORD
from aleph.core import es, es_index, url_for


def records_query(document_id, args, size=5, snippet_size=50):
    shoulds = []
    text = args.get('q', '').strip()
    if len(text):
        shoulds.append({
            'query_string': {
                'query': text,
                'fields': ['text^10', 'text_latin'],
                'default_operator': 'AND',
                'use_dis_max': True
            }
        })

    entities = Entity.by_id_set(args.getlist('entity'))
    for entity in entities.values():
        for term in entity.terms:
            shoulds.append({
                'multi_match': {
                    'query': term,
                    'type': "best_fields",
                    'fields': ['text^5', 'text_latin'],
                    'operator': 'AND'
                }
            })

    if not len(shoulds):
        return None

    q = {
        'bool': {
            'minimum_should_match': 1,
            'should': shoulds
        }
    }
    if document_id is not None:
        q['bool']['must'] = {
            'term': {'document_id': document_id}
        }

    try:
        snippet = int(args.get('snippet', snippet_size))
    except:
        snippet = snippet_size

    return {
        'size': size,
        'query': q,
        'highlight': {
            'fields': {
{'fraction_non_alphanumeric': 0.1330537552420892, 'fraction_numerical': 0.004956157072054899, 'mean_word_length': 1.6478304742684158, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7113572', 'n_tokens_mistral': 817, 'n_tokens_neox': 779, 'n_words': 178}