<commit_before># -*- coding: utf-8 -*-
from copy import deepcopy
import logging
import datetime
import operator

from pprint import pprint as pp

import pymongo

from scout.parse.case import parse_case
from scout.build.case import build_case
from scout.parse.variant.ids import parse_document_id
from scout.utils.algorithms import ui_score

from scout.exceptions import IntegrityError, ConfigError

LOG = logging.getLogger(__name__)

class CaseHandler(object):
    """Part of the pymongo adapter that handles cases and institutes"""

    def get_similar_cases(self, case_obj):
        """Take a case obj and return a iterable with the most phenotypically similar cases

        Args:
            case_obj(models.Case)

        Returns:
            scores(list(tuple)): Returns a list of tuples like (case_id, score) with the most
                                 similar case first
        """
        scores = {}
        set_1 = set()
        if not case_obj.get('phenotype_terms'):
            LOG.warning("No phenotypes could be found for case %s", case_obj['_id'])
            return None
        # Add all ancestors of all terms
        for term in case_obj['phenotype_terms']:
            hpo_term = self.hpo_term(term['phenotype_id'])
            if not hpo_term:
                continue
            set_1 = set_1.union(set(hpo_term.get('all_ancestors',[])))
        # Need to control what cases to look for here
        # Fetch all cases with phenotypes
        for case in self.cases(phenotype_terms=True, owner=case_obj['owner']):
            set_2 = set()
            if case['_id'] == case_obj['_id']:
                continue
            # Add all ancestors if all terms
            for term in case['phenotype_terms']:
                hpo_term = self.hpo_term(term['phenotype_id'])
                if not hpo_term:
                    continue
                set_2 = set_2.union(set(hpo_term.get('all_ancestors',[])))
            LOG.debug("Check phenotypic similarity of %s and %s", case_obj['_id'], case['_id'])
            scores[case['_id']] = ui_score(set_1, set_2)
        # Returns a list of tuples with highest score first
        return sorted(scores.items(), key=operator.itemgetter(1), reverse=True)

    def cases(self, owner=None, collaborator=None, query=None, skip_assigned=False,
              has_causatives=False, reruns=False, finished=False,
              research_requested=False, is_research=False, status=None,
              phenotype_terms=False, pinned=False, cohort=False, name_query=None,
              yield_query=False):
        """Fetches all cases from the backend.

        Args:
            collaborator(str): If collaborator should be considered
            owner(str): Query cases for specified case owner only
            query(dict): If a specific query is used
            skip_assigned(bool)
            has_causatives(bool)
            reruns(bool)
            finished(bool)
            research_requested(bool)
            is_research(bool)
            status(str)
            phenotype_terms(bool): Fetch all cases with phenotype terms
            pinned(bool): Fetch all cases with pinned variants
            name_query(str): Could be hpo term, HPO-group, user, part of display name,
                             part of inds or part of synopsis
            yield_query(bool): If true, only return mongo query dict for use in
                                compound querying.

        Returns:
            Cases ordered by date.
            If yield_query is True, does not pose query to db;
                instead returns corresponding query dict
                that can be reused in compound queries or for testing.
        """
        LOG.debug("Fetch all cases")
        query = query or {}
        order = None

        # Prioritize when both owner and collaborator params are present
        if collaborator and owner:
            collaborator = None

        if collaborator:
            LOG.debug("Use collaborator {0}".format(collaborator))
            query['collaborators'] = collaborator

        if owner:
            LOG.debug("Use owner {0}".format(owner))
            query['owner'] = owner

        if skip_assigned:
            query['assignees'] = {'$exists': False}

        if has_causatives:
            query['causatives'] = {'$exists': True, '$ne': []}

        if reruns:
            query['rerun_requested'] = True

        if status:
            query['status'] = status

        elif finished:
            query['status'] = {'$in': ['solved', 'archived']}

        if research_requested:
            query['research_requested'] = True

        if is_research:
            query['is_research'] = {'$exists': True, '$eq': True}

        if phenotype_terms:
            query['phenotype_terms'] = {'$exists': True, '$ne': []}

        if pinned:
            query['suspects'] = {'$exists': True, '$ne': []}

        if cohort:
            query['cohorts'] = {'$exists': True, '$ne': []}

        if name_query:
            name_value = name_query.split(':')[-1] # capture ant value provided after query descriptor
            users = self.user_collection.find({'name': {'$regex': name_query, '$options': 'i'}})
            if users.count() > 0:
                query['assignees'] = {'$in': [user['email'] for user in users]}
            elif name_query.startswith('HP:'):
                LOG.debug("HPO case query")
                if name_value:
                    query['phenotype_terms.phenotype_id'] = name_query
                else: # query for cases with no HPO terms
                    query['$or'] = [ {'phenotype_terms' : {'$size' : 0}}, {'phenotype_terms' : {'$exists' : False}} ]
            elif name_query.startswith('PG:'):
                LOG.debug("PG case query")
                if name_value:
                    phenotype_group_query = name_query.replace('PG:', 'HP:')
                    query['phenotype_groups.phenotype_id'] = phenotype_group_query
                else: # query for cases with no phenotype groups
                    query['$or'] = [ {'phenotype_groups' : {'$size' : 0}}, {'phenotype_groups' : {'$exists' : False}} ]
            elif name_query.startswith('similar:'):
                LOG.debug("Case HPO similarity query")
                if name_value:
                    # first, see that we can find a unique case with the given display name to match against for owner
                    if owner:
                        search_institute_id=owner
                    elif collaborator:
                        search_institute_id=collaborator
                    else:
                        raise ValueError("No owner or collaborator institute_id given.")
                    case_obj = self.case(display_name=name_value, institute_id=search_institute_id)
                    if case_obj:
                        LOG.debug("Search for cases similar to %s", case_obj.get('display_name'))
                        similar_cases = self.get_similar_cases(case_obj)
                        LOG.debug("Similar cases: %s",similar_cases)
                        if similar_cases:
                            similar_case_ids = []
                            order = []
                            for i in similar_cases:
                                similar_case_ids.append(i[0])
                                order.append(i[1])
                            query['_id'] = {'$in': similar_case_ids}
            elif name_query.startswith('synopsis:'):
                if name_value:
                    query['$text']={'$search':name_value}
                else: # query for cases with missing synopsis
                    query['synopsis'] = ''
            elif name_query.startswith('cohort:'):
                query['cohorts'] = name_value
            elif name_query.startswith('panel:'):
                query['panels'] = {'$elemMatch': {'panel_name': name_value,<commit_msg>Fix #1065: last piece, gene in pin/causative.<commit_after># -*- coding: utf-8 -*-
from copy import deepcopy
import logging
import datetime
import operator

from pprint import pprint as pp

import pymongo

from scout.parse.case import parse_case
from scout.build.case import build_case
from scout.parse.variant.ids import parse_document_id
from scout.utils.algorithms import ui_score

from scout.exceptions import IntegrityError, ConfigError

LOG = logging.getLogger(__name__)

class CaseHandler(object):
    """Part of the pymongo adapter that handles cases and institutes"""

    def get_similar_cases(self, case_obj):
        """Take a case obj and return a iterable with the most phenotypically similar cases

        Args:
            case_obj(models.Case)

        Returns:
            scores(list(tuple)): Returns a list of tuples like (case_id, score) with the most
                                 similar case first
        """
        scores = {}
        set_1 = set()
        if not case_obj.get('phenotype_terms'):
            LOG.warning("No phenotypes could be found for case %s", case_obj['_id'])
            return None
        # Add all ancestors of all terms
        for term in case_obj['phenotype_terms']:
            hpo_term = self.hpo_term(term['phenotype_id'])
            if not hpo_term:
                continue
            set_1 = set_1.union(set(hpo_term.get('all_ancestors',[])))
        # Need to control what cases to look for here
        # Fetch all cases with phenotypes
        for case in self.cases(phenotype_terms=True, owner=case_obj['owner']):
            set_2 = set()
            if case['_id'] == case_obj['_id']:
                continue
            # Add all ancestors if all terms
            for term in case['phenotype_terms']:
                hpo_term = self.hpo_term(term['phenotype_id'])
                if not hpo_term:
                    continue
                set_2 = set_2.union(set(hpo_term.get('all_ancestors',[])))
            LOG.debug("Check phenotypic similarity of %s and %s", case_obj['_id'], case['_id'])
            scores[case['_id']] = ui_score(set_1, set_2)
        # Returns a list of tuples with highest score first
        return sorted(scores.items(), key=operator.itemgetter(1), reverse=True)

    def cases(self, owner=None, collaborator=None, query=None, skip_assigned=False,
              has_causatives=False, reruns=False, finished=False,
              research_requested=False, is_research=False, status=None,
              phenotype_terms=False, pinned=False, cohort=False, name_query=None,
              yield_query=False):
        """Fetches all cases from the backend.

        Args:
            collaborator(str): If collaborator should be considered
            owner(str): Query cases for specified case owner only
            query(dict): If a specific query is used
            skip_assigned(bool)
            has_causatives(bool)
            reruns(bool)
            finished(bool)
            research_requested(bool)
            is_research(bool)
            status(str)
            phenotype_terms(bool): Fetch all cases with phenotype terms
            pinned(bool): Fetch all cases with pinned variants
            name_query(str): Could be hpo term, HPO-group, user, part of display name,
                             part of inds or part of synopsis
            yield_query(bool): If true, only return mongo query dict for use in
                                compound querying.

        Returns:
            Cases ordered by date.
            If yield_query is True, does not pose query to db;
                instead returns corresponding query dict
                that can be reused in compound queries or for testing.
        """
        LOG.debug("Fetch all cases")
        query = query or {}
        order = None

        # Prioritize when both owner and collaborator params are present
        if collaborator and owner:
            collaborator = None

        if collaborator:
            LOG.debug("Use collaborator {0}".format(collaborator))
            query['collaborators'] = collaborator

        if owner:
            LOG.debug("Use owner {0}".format(owner))
            query['owner'] = owner

        if skip_assigned:
            query['assignees'] = {'$exists': False}

        if has_causatives:
            query['causatives'] = {'$exists': True, '$ne': []}

        if reruns:
            query['rerun_requested'] = True

        if status:
            query['status'] = status

        elif finished:
            query['status'] = {'$in': ['solved', 'archived']}

        if research_requested:
            query['research_requested'] = True

        if is_research:
            query['is_research'] = {'$exists': True, '$eq': True}

        if phenotype_terms:
            query['phenotype_terms'] = {'$exists': True, '$ne': []}

        if pinned:
            query['suspects'] = {'$exists': True, '$ne': []}

        if cohort:
            query['cohorts'] = {'$exists': True, '$ne': []}

        if name_query:
            name_value = name_query.split(':')[-1] # capture ant value provided after query descriptor
            users = self.user_collection.find({'name': {'$regex': name_query, '$options': 'i'}})
            if users.count() > 0:
                query['assignees'] = {'$in': [user['email'] for user in users]}
            elif name_query.startswith('HP:'):
                LOG.debug("HPO case query")
                if name_value:
                    query['phenotype_terms.phenotype_id'] = name_query
                else: # query for cases with no HPO terms
                    query['$or'] = [ {'phenotype_terms' : {'$size' : 0}}, {'phenotype_terms' : {'$exists' : False}} ]
            elif name_query.startswith('PG:'):
                LOG.debug("PG case query")
                if name_value:
                    phenotype_group_query = name_query.replace('PG:', 'HP:')
                    query['phenotype_groups.phenotype_id'] = phenotype_group_query
                else: # query for cases with no phenotype groups
                    query['$or'] = [ {'phenotype_groups' : {'$size' : 0}}, {'phenotype_groups' : {'$exists' : False}} ]
            elif name_query.startswith('similar:'):
                LOG.debug("Case HPO similarity query")
                if name_value:
                    # first, see that we can find a unique case with the given display name to match against for owner
                    if owner:
                        search_institute_id=owner
                    elif collaborator:
                        search_institute_id=collaborator
                    else:
                        raise ValueError("No owner or collaborator institute_id given.")
                    case_obj = self.case(display_name=name_value, institute_id=search_institute_id)
                    if case_obj:
                        LOG.debug("Search for cases similar to %s", case_obj.get('display_name'))
                        similar_cases = self.get_similar_cases(case_obj)
                        LOG.debug("Similar cases: %s",similar_cases)
                        if similar_cases:
                            similar_case_ids = []
                            order = []
                            for i in similar_cases:
                                similar_case_ids.append(i[0])
                                order.append(i[1])
                            query['_id'] = {'$in': similar_case_ids}
            elif name_query.startswith('causative:'):
                if name_value:
                    cases_with_gene_doc=self.case_collection.aggregate([
                        { '$lookup':
                            {'from': 'variant',
                            'localField': 'causatives',
                            'foreignField': '_id',
                            'as': 'causative_variant'}
                        },
                        {'$match': {
                            'causative_variant.hgnc_symbols': name_value
                        }},
                        {'$project': {'_id': 1}}])
                    case_ids = [case['_id'] for case in cases_with_gene_doc]
                    query['_id'] = {'$in': case_ids}
            elif name_query.startswith('pinned:'):
                if name_value:
                    cases_with_gene_doc=self.case_collection.aggregate([
                        { '$lookup':
                            {'from': 'variant',
                            'localField': 'suspects',
                            'foreignField': '_id',
                            'as': 'suspect_variant'}
                        },
                        {'$match': {
                            'suspect_variant.hgnc_symbols': name_value
                        }},
                        {'$project': {'_id': 1}}])
                    case_ids = [case['_id'] for case in cases_with_gene_doc]
                    query['_id'] = {'$in': case_ids}
            elif name_query.startswith('synopsis:'):
                if name_value:
                    query['$text']={'$search':name_value}
                else: # query for cases with missing synopsis
                    query['synopsis'] = ''
            elif name_query.startswith('cohort:'):
                query['cohorts'] = name_value
            elif name_query.startswith('panel:'):
                query['panels'] = {'$elemMatch': {'panel_name': name_value,
{'fraction_non_alphanumeric': 0.1029555798813159, 'fraction_numerical': 0.002419773002246932, 'mean_word_length': 1.7356973995271867, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1047660', 'n_tokens_mistral': 4755, 'n_tokens_neox': 4449, 'n_words': 1231}