<commit_before>from pprint import pprint  # noqa
from elasticsearch.helpers import scan

from aleph.core import es, es_index
from aleph.search.result import SearchQueryResult
from aleph.search.parser import SearchQueryParser


class Query(object):
    RESULT_CLASS = SearchQueryResult
    DOC_TYPES = []
    RETURN_FIELDS = True
    TEXT_FIELDS = ['_all']
    MULTI_FIELDS = ['collection_id', 'schema', 'countries']
    SORT = {
        'default': ['_score']
    }

    def __init__(self, parser):
        self.parser = parser

    def get_text_query(self):
        if not self.parser.text:
            return {'match_all': {}}
        return {
            "simple_query_string": {
                "query": self.parser.text,
                "fields": self.TEXT_FIELDS,
                "default_operator": "AND"
            }
        }

    def get_filters(self, exclude=None):
        """Apply query filters from the user interface."""
        filters = []
        for field, values in self.parser.filters.items():
            if field == exclude:
                continue
            if field in ['id', '_id']:
                filters.append({'ids': {'values': list(values)}})
            elif field in self.MULTI_FIELDS:
                filters.append({'terms': {field: list(values)}})
            else:
                for value in values:
                    filters.append({'term': {field: value}})
        return filters

    def get_query(self):
        return {
            'bool': {
                'should': [],
                'must': [self.get_text_query()],
                'must_not': [],
                'filter': self.get_filters()
            }
        }

    def get_aggregations(self):
        """Aggregate the query in order to generate faceted results."""
        aggs = {}
        for name in self.parser.facet_names:
            facet = {
                name: {
                    'terms': {
                        'field': name,
                        'size': self.parser.facet_size
                    }
                }
            }
            # Fields to be excluded from their own aggregation. This is true
            # when a filter on a faceted fied is not supposed to affect the
            # selection of aggregations.
            if name in self.MULTI_FIELDS:
                if 'scoped' not in aggs:<commit_msg>Fix id and _id not being combined into a single filter<commit_after>from pprint import pprint  # noqa
from elasticsearch.helpers import scan

from aleph.core import es, es_index
from aleph.search.result import SearchQueryResult
from aleph.search.parser import SearchQueryParser


class Query(object):
    RESULT_CLASS = SearchQueryResult
    DOC_TYPES = []
    RETURN_FIELDS = True
    TEXT_FIELDS = ['_all']
    MULTI_FIELDS = ['collection_id', 'schema', 'countries']
    SORT = {
        'default': ['_score']
    }

    def __init__(self, parser):
        self.parser = parser

    def get_text_query(self):
        if not self.parser.text:
            return {'match_all': {}}
        return {
            "simple_query_string": {
                "query": self.parser.text,
                "fields": self.TEXT_FIELDS,
                "default_operator": "AND"
            }
        }

    def get_filters(self, exclude=None):
        """Apply query filters from the user interface."""
        filters = []
        id_values = []
        for field, values in self.parser.filters.items():
            if field == exclude:
                continue
            # Combine id or _id into one filter
            if field in ['id', '_id']:
                id_values.extend(values)
            elif field in self.MULTI_FIELDS:
                filters.append({'terms': {field: list(values)}})
            else:
                for value in values:
                    filters.append({'term': {field: value}})

        if id_values:
            filters.append({'ids': {'values': id_values}})

        return filters

    def get_query(self):
        return {
            'bool': {
                'should': [],
                'must': [self.get_text_query()],
                'must_not': [],
                'filter': self.get_filters()
            }
        }

    def get_aggregations(self):
        """Aggregate the query in order to generate faceted results."""
        aggs = {}
        for name in self.parser.facet_names:
            facet = {
                name: {
                    'terms': {
                        'field': name,
                        'size': self.parser.facet_size
                    }
                }
            }
            # Fields to be excluded from their own aggregation. This is true
            # when a filter on a faceted fied is not supposed to affect the
            # selection of aggregations.
            if name in self.MULTI_FIELDS:
                if 'scoped' not in aggs:
{'fraction_non_alphanumeric': 0.10213289581624282, 'fraction_numerical': 0.0, 'mean_word_length': 1.5587618048268626, 'pattern_counts': {'":': 8, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1163653', 'n_tokens_mistral': 1345, 'n_tokens_neox': 1287, 'n_words': 364}