<commit_before>"""
    Elasticsearch Query Result Transform
"""
from collections import defaultdict

from biothings.utils.common import dotdict


class ResultTransformException(Exception):
    pass

class ESResultTransform(object):
    ''' Class to transform the results of the Elasticsearch query generated prior in the pipeline.
    This contains the functions to extract the final document from the elasticsearch query result
    in `Elasticsearch Query`_.  This also contains the code to flatten a document etc.
    '''

    def __init__(self, web_settings):

        # license appending settings
        self.source_licenses = web_settings.metadata.biothing_licenses
        self.license_transform = web_settings.LICENSE_TRANSFORM

        # mapping transform
        self.field_notes = web_settings.fieldnote.get_field_notes()
        self.excluded_keys = web_settings.AVAILABLE_FIELDS_EXCLUDED

    @classmethod
    def traverse(cls, obj, leaf_node=False):
        """
        Output path-dictionary pairs. For example, input:
        {
            'exac_nontcga': {'af': 0.00001883},
            'gnomad_exome': {'af': {'af': 0.0000119429, 'af_afr': 0.000123077}},
            'snpeff': {'ann': [{'effect': 'intron_variant',
                                'feature_id': 'NM_014672.3'},
                               {'effect': 'intron_variant',
                                'feature_id': 'NM_001256678.1'}]}
        }
        will be translated to a generator:
        (
            ("exac_nontcga", {"af": 0.00001883}),
            ("gnomad_exome.af", {"af": 0.0000119429, "af_afr": 0.000123077}),
            ("gnomad_exome", {"af": {"af": 0.0000119429, "af_afr": 0.000123077}}),
            ("snpeff.ann", {"effect": "intron_variant", "feature_id": "NM_014672.3"}),
            ("snpeff.ann", {"effect": "intron_variant", "feature_id": "NM_001256678.1"}),
            ("snpeff.ann", [{ ... },{ ... }]),
            ("snpeff", {"ann": [{ ... },{ ... }]}),
            ('', {'exac_nontcga': {...}, 'gnomad_exome': {...}, 'snpeff': {...}})
        )
        or when traversing leaf nodes:
        (
            ('exac_nontcga.af', 0.00001883),
            ('gnomad_exome.af.af', 0.0000119429),
            ('gnomad_exome.af.af_afr', 0.000123077),
            ('snpeff.ann.effect', 'intron_variant'),
            ('snpeff.ann.feature_id', 'NM_014672.3'),
            ('snpeff.ann.effect', 'intron_variant'),
            ('snpeff.ann.feature_id', 'NM_001256678.1')
        )
        """
        if isinstance(obj, dict):  # path level increases
            for key in obj:
                for sub_path, val in cls.traverse(obj[key], leaf_node):
                    yield '.'.join((key, sub_path)).strip('.'), val
            if not leaf_node:
                yield '', obj
        elif isinstance(obj, list):  # does not affect path
            for item in obj:
                for sub_path, val in cls.traverse(item, leaf_node):
                    yield sub_path, val
            if not leaf_node or not obj:  # [] count as a leaf node
                yield '', obj
        elif leaf_node:  # including str, int, float, and *None*.
            yield '', obj

    def transform(self, response, options):
        """
        Transform the query response to a user-friendly structure.

        Options:
            dotfield: flatten a dictionary using dotfield notation
            _sorted: sort keys alaphabetically in ascending order
            always_list: ensure the fields specified are lists or wrapped in a list
            allow_null: ensure the fields specified are present in the result,
                        the fields may be provided as type None or [].
            # only related to multiqueries
            template: base dict for every result, for example: {"success": true}
            templates: a different base for every result, replaces the setting above
            template_hit: a dict to update every positive hit result, default: {"found": true}
            template_miss: a dict to update every query with no hit, default: {"found": false}

        """
        if not isinstance(options, dotdict):
            options = dotdict(options)
        if isinstance(response, list):
            responses_ = []
            template = options.pop('template', {})
            templates = options.pop('templates', [template]*len(response))
            template_hit = options.pop('template_hit', dict(found=True))
            template_miss = options.pop('template_miss', dict(found=False))
            responses = [self.transform(res, options) for res in response]
            for res_, res in zip(templates, responses):
                if not res.get('hits'):
                    res_.update(template_miss)
                    responses_.append(res_)
                else:
                    for hit in res['hits']:
                        hit_ = dict(res_)<commit_msg>Refine documentation<commit_after>"""
    Elasticsearch Query Result Transform
"""
from collections import defaultdict

from biothings.utils.common import dotdict


class ResultTransformException(Exception):
    pass

class ESResultTransform(object):
    ''' Class to transform the results of the Elasticsearch query generated prior in the pipeline.
    This contains the functions to extract the final document from the elasticsearch query result
    in `Elasticsearch Query`_.  This also contains the code to flatten a document etc.
    '''

    def __init__(self, web_settings):

        # license appending settings
        self.source_licenses = web_settings.metadata.biothing_licenses
        self.license_transform = web_settings.LICENSE_TRANSFORM

        # mapping transform
        self.field_notes = web_settings.fieldnote.get_field_notes()
        self.excluded_keys = web_settings.AVAILABLE_FIELDS_EXCLUDED

    @classmethod
    def traverse(cls, obj, leaf_node=False):
        """
        Output path-dictionary pairs. For example, input:
        {
            'exac_nontcga': {'af': 0.00001883},
            'gnomad_exome': {'af': {'af': 0.0000119429, 'af_afr': 0.000123077}},
            'snpeff': {'ann': [{'effect': 'intron_variant',
                                'feature_id': 'NM_014672.3'},
                               {'effect': 'intron_variant',
                                'feature_id': 'NM_001256678.1'}]}
        }
        will be translated to a generator:
        (
            ("exac_nontcga", {"af": 0.00001883}),
            ("gnomad_exome.af", {"af": 0.0000119429, "af_afr": 0.000123077}),
            ("gnomad_exome", {"af": {"af": 0.0000119429, "af_afr": 0.000123077}}),
            ("snpeff.ann", {"effect": "intron_variant", "feature_id": "NM_014672.3"}),
            ("snpeff.ann", {"effect": "intron_variant", "feature_id": "NM_001256678.1"}),
            ("snpeff.ann", [{ ... },{ ... }]),
            ("snpeff", {"ann": [{ ... },{ ... }]}),
            ('', {'exac_nontcga': {...}, 'gnomad_exome': {...}, 'snpeff': {...}})
        )
        or when traversing leaf nodes:
        (
            ('exac_nontcga.af', 0.00001883),
            ('gnomad_exome.af.af', 0.0000119429),
            ('gnomad_exome.af.af_afr', 0.000123077),
            ('snpeff.ann.effect', 'intron_variant'),
            ('snpeff.ann.feature_id', 'NM_014672.3'),
            ('snpeff.ann.effect', 'intron_variant'),
            ('snpeff.ann.feature_id', 'NM_001256678.1')
        )
        """
        if isinstance(obj, dict):  # path level increases
            for key in obj:
                for sub_path, val in cls.traverse(obj[key], leaf_node):
                    yield '.'.join((key, sub_path)).strip('.'), val
            if not leaf_node:
                yield '', obj
        elif isinstance(obj, list):  # does not affect path
            for item in obj:
                for sub_path, val in cls.traverse(item, leaf_node):
                    yield sub_path, val
            if not leaf_node or not obj:  # [] count as a leaf node
                yield '', obj
        elif leaf_node:  # including str, int, float, and *None*.
            yield '', obj

    def transform(self, response, options):
        """
        Transform the query response to a user-friendly structure.

        Options:
            dotfield: flatten a dictionary using dotfield notation
            _sorted: sort keys alaphabetically in ascending order
            always_list: ensure the fields specified are lists or wrapped in a list
            allow_null: ensure the fields specified are present in the result,
                        the fields may be provided as type None or [].
            biothing_type: result document type to apply customized transformation.
                        for example, add license field basing on document type's metadata.
            # only related to multiqueries
            template: base dict for every result, for example: {"success": true}
            templates: a different base for every result, replaces the setting above
            template_hit: a dict to update every positive hit result, default: {"found": true}
            template_miss: a dict to update every query with no hit, default: {"found": false}
        """
        if not isinstance(options, dotdict):
            options = dotdict(options)
        if isinstance(response, list):
            responses_ = []
            template = options.pop('template', {})
            templates = options.pop('templates', [template]*len(response))
            template_hit = options.pop('template_hit', dict(found=True))
            template_miss = options.pop('template_miss', dict(found=False))
            responses = [self.transform(res, options) for res in response]
            for res_, res in zip(templates, responses):
                if not res.get('hits'):
                    res_.update(template_miss)
                    responses_.append(res_)
                else:
                    for hit in res['hits']:
                        hit_ = dict(res_)
{'fraction_non_alphanumeric': 0.11754684838160136, 'fraction_numerical': 0.032468183184687846, 'mean_word_length': 2.0510547233262, 'pattern_counts': {'":': 28, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4992972', 'n_tokens_mistral': 2978, 'n_tokens_neox': 2717, 'n_words': 782}