<commit_before>"""
    Elasticsearch Query Execution
"""
import asyncio

from elasticsearch import NotFoundError, RequestError, TransportError

from biothings.web.api.handler import BadRequest, EndRequest


class ESQuery(object):
    '''
    Execute an Elasticsearch query
    '''

    def __init__(self, web_settings):

        self.client = web_settings.get_async_es_client()

        # es indices
        self.indices = web_settings.ES_INDICES
        self.default_index = web_settings.ES_INDEX
        self.default_type = web_settings.ES_DOC_TYPE

        # for scroll queries
        self.scroll_time = web_settings.ES_SCROLL_TIME
        self.scroll_size = web_settings.ES_SCROLL_SIZE

    async def execute(self, query, options):
        '''
        Execute the corresponding query.
        May override to add more. Handle uncaught exceptions.
        Must return an awaitable.
        '''
        if options.scroll_id:
            try:
                res = await self.client.scroll(
                    scroll_id=options.scroll_id,
                    scroll=self.scroll_time)
            except (NotFoundError, RequestError, TransportError):
                raise BadRequest(reason="Invalid or stale scroll_id.")
            else:
                if not res['hits']['hits']:
                    raise EndRequest(reason="No more results to return.")
                return res

        if query:
            biothing_type = options.pop('biothing_type', None) or self.default_type
            query = query.index(self.indices.get(biothing_type, self.default_index))

            if options.pop('fetch_all', False):
                query = query.params(scroll=self.scroll_time)
                query = query.extra(size=self.scroll_size)

            if options.sort:  # accept '-' prefixed field names
                query = query.sort(*options.pop('sort'))

            try:
                return await self.dsl_query(query, **options)
            except RequestError as err:
                raise BadRequest(root_cause=err.info['error']['root_cause'][0]['reason'])

        return asyncio.sleep(0, {})

    async def dsl_query(self, query, **options):

        query = query.extra(**options)
        res = await query.using(self.client).execute()
        if isinstance(res, list):
            return [res_.to_dict() for res_ in res]
        return res.to_dict()<commit_msg>put es options back to query builder<commit_after>"""
    Elasticsearch Query Execution
"""
import asyncio

from elasticsearch import NotFoundError, RequestError, TransportError

from biothings.web.api.handler import BadRequest, EndRequest


class ESQuery(object):
    '''
    Execute an Elasticsearch query
    '''

    def __init__(self, web_settings):

        self.client = web_settings.get_async_es_client()

        # es indices
        self.indices = web_settings.ES_INDICES
        self.default_index = web_settings.ES_INDEX
        self.default_type = web_settings.ES_DOC_TYPE

        # for scroll queries
        self.scroll_time = web_settings.ES_SCROLL_TIME
        self.scroll_size = web_settings.ES_SCROLL_SIZE

    async def execute(self, query, options):
        '''
        Execute the corresponding query.
        May override to add more. Handle uncaught exceptions.
        Must return an awaitable.
        '''
        if options.scroll_id:
            try:
                res = await self.client.scroll(
                    scroll_id=options.scroll_id,
                    scroll=self.scroll_time)
            except (NotFoundError, RequestError, TransportError):
                raise BadRequest(reason="Invalid or stale scroll_id.")
            else:
                if not res['hits']['hits']:
                    raise EndRequest(reason="No more results to return.")
                return res

        if query:
            biothing_type = options.pop('biothing_type', None) or self.default_type
            query = query.index(self.indices.get(biothing_type, self.default_index))

            if options.pop('fetch_all', False):
                query = query.params(scroll=self.scroll_time)
                query = query.extra(size=self.scroll_size)
            try:
                res = await query.using(self.client).execute()
            except RequestError as err:  # TODO
                raise BadRequest(root_cause=err.info['error']['root_cause'][0]['reason'])
            else:
                if isinstance(res, list):
                    return [res_.to_dict() for res_ in res]
                return res.to_dict()

        return asyncio.sleep(0, {})

{'fraction_non_alphanumeric': 0.0906707450294953, 'fraction_numerical': 0.0008739348918505571, 'mean_word_length': 2.1058344640434195, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1182332', 'n_tokens_mistral': 1250, 'n_tokens_neox': 1196, 'n_words': 292}