<commit_before># -*- coding: utf-8 -*-
from __future__ import unicode_literals

import copy
import logging

from functools import partial

from flask import request
from werkzeug.urls import Href

from elasticsearch_dsl import Search, FacetedSearch, Q
from elasticsearch_dsl.aggs import Bucket, Pipeline
from elasticsearch_dsl.query import FunctionScore

from udata.search import es, DEFAULT_PAGE_SIZE
from udata.search.result import SearchResult


log = logging.getLogger(__name__)


class SearchQuery(FacetedSearch):
    adapter = None
    analyzer = None
    boosters = None
    fuzzy = False
    match_type = None
    model = None

    def __init__(self, params):
        self.extract_sort(params)
        self.extract_pagination(params)
        q = params.pop('q', '')

        params = self.clean_parameters(params)

        super(SearchQuery, self).__init__(q, params)

    def clean_parameters(self, params):
        '''Only keep known parameters'''
        return {k: v for k, v in params.items() if k in self.adapter.facets}

    def extract_sort(self, params):
        '''Extract and build sort query from parameters'''
        sorts = params.pop('sort', [])
        sorts = [sorts] if isinstance(sorts, basestring) else sorts
        sorts = [(s[1:], 'desc')
                 if s.startswith('-') else (s, 'asc')
                 for s in sorts]
        self.sorts = [
            {self.adapter.sorts[s]: d}
            for s, d in sorts if s in self.adapter.sorts
        ]

    def extract_pagination(self, params):
        '''Extract and build pagination from parameters'''
        try:
            params_page = int(params.pop('page', 1) or 1)
            self.page = max(params_page, 1)
        except:
            # Failsafe, if page cannot be parsed, we falback on first page
            self.page = 1
        try:
            params_page_size = params.pop('page_size', DEFAULT_PAGE_SIZE)
            self.page_size = int(params_page_size or DEFAULT_PAGE_SIZE)
        except:
            # Failsafe, if page_size cannot be parsed, we falback on default
            self.page_size = DEFAULT_PAGE_SIZE
        self.page_start = (self.page - 1) * self.page_size
        self.page_end = self.page_start + self.page_size

    def aggregate(self, search):
        """
        Add aggregations representing the facets selected
        """
        for f, facet in self.facets.items():
            agg = facet.get_aggregation()
            if isinstance(agg, Bucket):
                search.aggs.bucket(f, agg)
            elif isinstance(agg, Pipeline):
                search.aggs.pipeline(f, agg)
            else:
                search.aggs.metric(f, agg)

    def filter(self, search):
        '''
        Perform filtering instead of default post-filtering.
        '''
        if not self._filters:
            return search
        filters = Q('match_all')
        for f in self._filters.values():
            filters &= f
        return search.filter(filters)

    def search(self):
        """
        Construct the Search object.
        """
        s = Search(doc_type=self.doc_types, using=es.client,
                   index=es.index_name)
        # don't return any fields, just the metadata
        s = s.fields([])
        # Sort from parameters
        s = s.sort(*self.sorts)
        # Paginate from parameters
        s = s[self.page_start:self.page_end]
        # Same construction as parent class
        # Allows to give the same signature as simple search
        # ie. Response(data) instead of Response(search, data)
        return s.response_class(partial(SearchResult, self))

    def query(self, search, query):
        '''
        Customize the search query if necessary.<commit_msg>Perform a Bool(must=filter) query instead of Bool(filter) for facets<commit_after># -*- coding: utf-8 -*-
from __future__ import unicode_literals

import copy
import logging

from functools import partial

from flask import request
from werkzeug.urls import Href

from elasticsearch_dsl import Search, FacetedSearch, Q
from elasticsearch_dsl.aggs import Bucket, Pipeline
from elasticsearch_dsl.query import FunctionScore

from udata.search import es, DEFAULT_PAGE_SIZE
from udata.search.result import SearchResult


log = logging.getLogger(__name__)


class SearchQuery(FacetedSearch):
    adapter = None
    analyzer = None
    boosters = None
    fuzzy = False
    match_type = None
    model = None

    def __init__(self, params):
        self.extract_sort(params)
        self.extract_pagination(params)
        q = params.pop('q', '')

        params = self.clean_parameters(params)

        super(SearchQuery, self).__init__(q, params)

    def clean_parameters(self, params):
        '''Only keep known parameters'''
        return {k: v for k, v in params.items() if k in self.adapter.facets}

    def extract_sort(self, params):
        '''Extract and build sort query from parameters'''
        sorts = params.pop('sort', [])
        sorts = [sorts] if isinstance(sorts, basestring) else sorts
        sorts = [(s[1:], 'desc')
                 if s.startswith('-') else (s, 'asc')
                 for s in sorts]
        self.sorts = [
            {self.adapter.sorts[s]: d}
            for s, d in sorts if s in self.adapter.sorts
        ]

    def extract_pagination(self, params):
        '''Extract and build pagination from parameters'''
        try:
            params_page = int(params.pop('page', 1) or 1)
            self.page = max(params_page, 1)
        except:
            # Failsafe, if page cannot be parsed, we falback on first page
            self.page = 1
        try:
            params_page_size = params.pop('page_size', DEFAULT_PAGE_SIZE)
            self.page_size = int(params_page_size or DEFAULT_PAGE_SIZE)
        except:
            # Failsafe, if page_size cannot be parsed, we falback on default
            self.page_size = DEFAULT_PAGE_SIZE
        self.page_start = (self.page - 1) * self.page_size
        self.page_end = self.page_start + self.page_size

    def aggregate(self, search):
        """
        Add aggregations representing the facets selected
        """
        for f, facet in self.facets.items():
            agg = facet.get_aggregation()
            if isinstance(agg, Bucket):
                search.aggs.bucket(f, agg)
            elif isinstance(agg, Pipeline):
                search.aggs.pipeline(f, agg)
            else:
                search.aggs.metric(f, agg)

    def filter(self, search):
        '''
        Perform filtering instead of default post-filtering.
        '''
        if not self._filters:
            return search
        for f in self._filters.values():
            search = search.query(f)  # this will make a Bool(must=filter)
        return search

    def search(self):
        """
        Construct the Search object.
        """
        s = Search(doc_type=self.doc_types, using=es.client,
                   index=es.index_name)
        # don't return any fields, just the metadata
        s = s.fields([])
        # Sort from parameters
        s = s.sort(*self.sorts)
        # Paginate from parameters
        s = s[self.page_start:self.page_end]
        # Same construction as parent class
        # Allows to give the same signature as simple search
        # ie. Response(data) instead of Response(search, data)
        return s.response_class(partial(SearchResult, self))

    def query(self, search, query):
        '''
        Customize the search query if necessary.
{'fraction_non_alphanumeric': 0.09312549960031974, 'fraction_numerical': 0.0018651745270450307, 'mean_word_length': 2.1876857749469214, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4294763', 'n_tokens_mistral': 2155, 'n_tokens_neox': 2096, 'n_words': 638}