<commit_before>from biothings.www.api.helper import BaseHandler
from biothings.utils.common import dotdict, is_str
import re
import logging

class BaseESRequestHandler(BaseHandler):
    # override these in child class
    control_kwargs = {}
    es_kwargs = {}
    esqb_kwargs = {}
    transform_kwargs = {}

    def initialize(self, web_settings):
        ''' Initializations common to all ES Request Handlers here '''
        super(BaseESRequestHandler, self).initialize(web_settings)

    def _return_data_and_track(self, data, ga_event_data={}, rawquery=False):
        ''' Small function to return a chunk of data and send a google analytics tracking request.'''
        if rawquery:
            self.return_raw_query_json(data)
        else:
            self.return_json(data)
        self.ga_track(event=self.ga_event_object(ga_event_data))
        return

    def return_raw_query_json(self, query):
        _ret = query.get('body', {'GET': query.get('bid')})
        if is_str(_ret) and len(_ret.split('\n')) > 1:
            self.return_json({'body': _ret})
        else:
            self.return_json(_ret)  

    def _should_sanitize(self, param, kwargs):
        return ((param in kwargs) and (param in self.kwarg_settings))

    def _sanitize_source_param(self, kwargs):
        if self._should_sanitize('_source', kwargs):
            if len(kwargs['_source']) == 1 and kwargs['_source'][0].lower() == 'all':
                kwargs['_source'] = True
        return kwargs

    def _sanitize_sort_param(self, kwargs):
        if self._should_sanitize('sort', kwargs):
            dirty_sort = kwargs['sort']
            kwargs['sort'] = [{field[1:]:{"order":"desc"}} if field.startswith('-') 
                else {field:{"order":"asc"}} for field in dirty_sort]
        return kwargs

    def _sanitize_size_param(self, kwargs):
        # cap size
        if self._should_sanitize('size', kwargs):
            kwargs['size'] = kwargs['size'] if (kwargs['size'] < 
                self.web_settings.ES_SIZE_CAP) else self.web_settings.ES_SIZE_CAP
        return kwargs

    def get_cleaned_options(self, kwargs):
        ''' Get options for handlers using ES requests '''
        options = dotdict()

        # split kwargs into one (or more) of 4 categories:
        #   * control_kwargs:  kwargs that control aspects of the handler's pipeline (e.g. raw, rawquery)
        #   * es_kwargs: kwargs that go directly to the ES query (e.g. fields, size, ...)
        #   * esqb_kwargs: kwargs that go directly to the ESQueryBuilder instance
        #   * transform_kwargs: kwargs that go directly to the response transformer (e.g. jsonld, dotfield)

        for kwarg_category in ["control_kwargs", "es_kwargs", "esqb_kwargs", "transform_kwargs"]:
            options.setdefault(kwarg_category, dotdict())
            for option, settings in getattr(self, kwarg_category, {}).items():
                if kwargs.get(option, None) or settings.get('default', None) is not None:
                    options.get(kwarg_category).setdefault(option, kwargs.get(option, settings['default']))
        return options

    def _sanitize_params(self, kwargs):
        kwargs = super(BaseESRequestHandler, self)._sanitize_params(kwargs)
        kwargs = self._sanitize_source_param(kwargs)
        kwargs = self._sanitize_size_param(kwargs)
        kwargs = self._sanitize_sort_param(kwargs)
        return kwargs
    
    def _get_es_index(self, options):
        ''' Override to change query index for this request. '''
        return self.web_settings.ES_INDEX<commit_msg>Adding sanitization for facets.<commit_after>from biothings.www.api.helper import BaseHandler
from biothings.utils.common import dotdict, is_str
import re
import logging

class BaseESRequestHandler(BaseHandler):
    # override these in child class
    control_kwargs = {}
    es_kwargs = {}
    esqb_kwargs = {}
    transform_kwargs = {}

    def initialize(self, web_settings):
        ''' Initializations common to all ES Request Handlers here '''
        super(BaseESRequestHandler, self).initialize(web_settings)

    def _return_data_and_track(self, data, ga_event_data={}, rawquery=False):
        ''' Small function to return a chunk of data and send a google analytics tracking request.'''
        if rawquery:
            self.return_raw_query_json(data)
        else:
            self.return_json(data)
        self.ga_track(event=self.ga_event_object(ga_event_data))
        return

    def return_raw_query_json(self, query):
        _ret = query.get('body', {'GET': query.get('bid')})
        if is_str(_ret) and len(_ret.split('\n')) > 1:
            self.return_json({'body': _ret})
        else:
            self.return_json(_ret)  

    def _should_sanitize(self, param, kwargs):
        return ((param in kwargs) and (param in self.kwarg_settings))

    def _sanitize_source_param(self, kwargs):
        if self._should_sanitize('_source', kwargs):
            if len(kwargs['_source']) == 1 and kwargs['_source'][0].lower() == 'all':
                kwargs['_source'] = True
        return kwargs

    def _sanitize_sort_param(self, kwargs):
        if self._should_sanitize('sort', kwargs):
            dirty_sort = kwargs['sort']
            kwargs['sort'] = [{field[1:]:{"order":"desc"}} if field.startswith('-') 
                else {field:{"order":"asc"}} for field in dirty_sort]
        return kwargs

    def _sanitize_size_param(self, kwargs):
        # cap size
        if self._should_sanitize('size', kwargs):
            kwargs['size'] = kwargs['size'] if (kwargs['size'] < 
                self.web_settings.ES_SIZE_CAP) else self.web_settings.ES_SIZE_CAP
        return kwargs

    def _sanitize_aggs_param(self, kwargs):
        if self._should_sanitize('aggs', kwargs):
            kwargs['aggs'] = dict([(field, {"terms": {"field": field}}) for field in kwargs['aggs']])
        return kwargs

    def get_cleaned_options(self, kwargs):
        ''' Get options for handlers using ES requests '''
        options = dotdict()

        # split kwargs into one (or more) of 4 categories:
        #   * control_kwargs:  kwargs that control aspects of the handler's pipeline (e.g. raw, rawquery)
        #   * es_kwargs: kwargs that go directly to the ES query (e.g. fields, size, ...)
        #   * esqb_kwargs: kwargs that go directly to the ESQueryBuilder instance
        #   * transform_kwargs: kwargs that go directly to the response transformer (e.g. jsonld, dotfield)

        for kwarg_category in ["control_kwargs", "es_kwargs", "esqb_kwargs", "transform_kwargs"]:
            options.setdefault(kwarg_category, dotdict())
            for option, settings in getattr(self, kwarg_category, {}).items():
                if kwargs.get(option, None) or settings.get('default', None) is not None:
                    options.get(kwarg_category).setdefault(option, kwargs.get(option, settings['default']))
        return options

    def _sanitize_params(self, kwargs):
        kwargs = super(BaseESRequestHandler, self)._sanitize_params(kwargs)
        kwargs = self._sanitize_source_param(kwargs)
        kwargs = self._sanitize_size_param(kwargs)
        kwargs = self._sanitize_sort_param(kwargs)
        kwargs = self._sanitize_aggs_param(kwargs)
        return kwargs
    
    def _get_es_index(self, options):
        ''' Override to change query index for this request. '''
        return self.web_settings.ES_INDEX
{'fraction_non_alphanumeric': 0.1117092866756393, 'fraction_numerical': 0.0013458950201884253, 'mean_word_length': 2.8723293381969777, 'pattern_counts': {'":': 6, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4635419', 'n_tokens_mistral': 2145, 'n_tokens_neox': 2170, 'n_words': 557}