<commit_before>from xbrowse_server.phenotips.reporting_utilities import get_phenotypes_entered_for_individual
import hashlib
import datetime
from django.conf import settings
from xbrowse_server.base.models import Individual
from django.shortcuts import get_object_or_404
from xbrowse_server.base.models import Project, Family
from xbrowse_server.base.models import ProjectTag, VariantTag
from xbrowse_server.mall import get_datastore
import time
from xbrowse_server.mall import get_reference
import json
from slacker import Slacker
from collections import defaultdict, namedtuple
from xbrowse_server.gene_lists.models import GeneList
from tqdm import tqdm
from reference_data.models import HumanPhenotypeOntology
import logging
from django.core.exceptions import ObjectDoesNotExist
from seqr.models import Project as SeqrProject
from django.core.exceptions import ObjectDoesNotExist

logger = logging.getLogger()

def get_all_clinical_data_for_family(project_id,family_id,indiv_id):
    """
        Gets phenotype and genotype data for this individual
        Args:
            family_id: id of family
        Returns:
            A JSON object as per MME spec of a patient
    """
    project = get_object_or_404(Project, project_id=project_id)
    
    #get contact information if available
    seqr_project = SeqrProject.objects.get(name=project_id)
    seqr_project=SeqrProject(name=project_id)
    seqr_project.is_mme_enabled=True
    seqr_project.mme_primary_data_owner=settings.MME_CONTACT_NAME
    seqr_project.mme_contact_url=settings.MME_CONTACT_HREF
    seqr_project.save()

    #species (only human for now) till seqr starts tracking species
    species="NCBITaxon:9606"
    
    contact={
             "name":seqr_project.mme_primary_data_owner,
             "institution" : settings.MME_CONTACT_INSTITUTION,
             "href" : seqr_project.mme_contact_url
             }

    #contact (this should be set in settings
    #href=settings.MME_CONTACT_HREF
    #if settings.MME_PATIENT_PRIMARY_DATA_OWNER[project_id]["email"] != "":
    #    href = href + ',' + settings.MME_PATIENT_PRIMARY_DATA_OWNER[project_id]["email"]
    
    #contact={
    #         "name":settings.MME_CONTACT_NAME + ' (data owner: ' + settings.MME_PATIENT_PRIMARY_DATA_OWNER[project_id]["PI"] + ')',
    #         "institution" : settings.MME_CONTACT_INSTITUTION,
    #         "href" : href
    #         }
        
    #genomicFeatures section
    genomic_features=[]
    variants=[]
    project_tags = ProjectTag.objects.filter(project__project_id=project_id)
    for project_tag in project_tags:
        variant_tags = VariantTag.objects.filter(project_tag=project_tag)
        for variant_tag in variant_tags:    
            if family_id == variant_tag.toJSON()['family']:
                variant = get_datastore(project.project_id).get_single_variant(
                        project.project_id,
                        variant_tag.toJSON()['family'],
                        variant_tag.xpos,
                        variant_tag.ref,
                        variant_tag.alt,
                )
                if variant is None:
                    logging.info("Variant no longer called in this family (did the callset version change?)")
                    continue
                variants.append({"variant":variant.toJSON(),
                                 "tag":project_tag.title,
                                 "family":variant_tag.family.toJSON(),
                                 "tag_name":variant_tag.toJSON()['tag']
                                 })
    #start compiling a matchmaker-esque data structure to send back
    genomic_features=[]
    for variant in variants:
        start = variant['variant']['pos']
        reference_bases = variant['variant']['ref']
        alternate_bases = variant['variant']['alt']
        end = int(variant['variant']['pos_end']) #int and long are unified in python
        reference_name = variant['variant']['chr'].replace('chr','')        
        #now we have more than 1 gene associated to these VAR postions,
        #so we will associate that information to each gene symbol
        for i,gene_id in enumerate(variant['variant']['gene_ids']):
            genomic_feature = {}
            genomic_feature['gene'] ={"id": gene_id }
            genomic_feature['variant']={
                                        'assembly':settings.GENOME_ASSEMBLY_NAME,
                                        'referenceBases':reference_bases,
                                        'alternateBases':alternate_bases,
                                        'start':start,
                                        'end':end,
                                        'referenceName':reference_name
                                        }
            genomic_feature['zygosity'] = variant['variant']['genotypes'][indiv_id]['num_alt']
            gene_symbol=""
            if gene_id != "":
                gene = get_reference().get_gene(gene_id)
                gene_symbol = gene['symbol']

            genomic_feature['auxiliary']={
                                          "tag_name":variant['tag_name'],
                                          "gene_symbol":gene_symbol
                                          }
            genomic_features.append(genomic_feature) 
    
    #Find phenotype information
    indiv = Individual.objects.get(indiv_id=indiv_id,project=project)
    phenotypes_entered = get_phenotypes_entered_for_individual(project_id,indiv.phenotips_id)
    #need to eventually support "FEMALE"|"MALE"|"OTHER"|"MIXED_SAMPLE"|"NOT_APPLICABLE",
    #as of now PhenoTips only has M/F
    sex="NOT_APPLICABLE"
    if "M" == indiv.gender:
        sex="MALE"
    if "F" == indiv.gender:
        sex="FEMALE"
    features=[]
    if phenotypes_entered.has_key('features'):
        #as of now non-standard features ('nonstandard_features') without HPO
        #terms cannot be sent to MME
        for f in phenotypes_entered['features']:
            features.append({
                "id":f['id'],
                "observed":f['observed'],
                "label":f['label']})
     
    #--depracating obfuscation as per discussion on slack and green light by @dgmacarthur       
    #make a unique hash to represent individual in MME for MME_ID
    #h = hashlib.md5()
    #h.update(indiv.indiv_id)
    #id=h.hexdigest()
    #label=id #using ID as label
    
    id=indiv.indiv_id
    label=indiv.indiv_id

    #add new patient to affected patients
    affected_patient={"id":id,
                      "species":species,
                      "label":label,
                      "contact":contact,
                      "features":features,
                      "sex":sex,
                      "genomicFeatures":genomic_features
                      }
                            
    #map to put into mongo
    time_stamp = datetime.datetime.fromtimestamp(time.time()).strftime('%Y_%m_%d_%H_%M_%S')
    detailed_id_map={"generated_on": time_stamp,
         "project_id":project_id,
         "family_id":family_id,
         "individual_id":indiv.indiv_id,
         "mme_id":id,
         "individuals_used_for_phenotypes":affected_patient}
    return detailed_id_map,affected_patient
            


def is_a_valid_patient_structure(patient_struct):
    """
    Checks to see if the input patient data structure has all the
    data/fields required by the MME
    Args:
        patient structure
    Returns:
        True if valid
    TODO:
        This function needs improvement and checks on field naming and other required values.
    """
    submission_validity={"status":True, "reason":""}
    #check if all gene IDs are present<commit_msg>refactoring, swapping in settings constants with the reading seqr.project models, cleaning up some code<commit_after>from xbrowse_server.phenotips.reporting_utilities import get_phenotypes_entered_for_individual
import hashlib
import datetime
from django.conf import settings
from xbrowse_server.base.models import Individual
from django.shortcuts import get_object_or_404
from xbrowse_server.base.models import Project, Family
from xbrowse_server.base.models import ProjectTag, VariantTag
from xbrowse_server.mall import get_datastore
import time
from xbrowse_server.mall import get_reference
import json
from slacker import Slacker
from collections import defaultdict, namedtuple
from xbrowse_server.gene_lists.models import GeneList
from tqdm import tqdm
from reference_data.models import HumanPhenotypeOntology
import logging
from django.core.exceptions import ObjectDoesNotExist
from seqr.models import Project as SeqrProject
from seqr.models import Dataset as Dataset
from django.core.exceptions import ObjectDoesNotExist

logger = logging.getLogger()

def get_all_clinical_data_for_family(project_id,family_id,indiv_id):
    """
        Gets phenotype and genotype data for this individual
        Args:
            family_id: id of family
        Returns:
            A JSON object as per MME spec of a patient
    """
    project = get_object_or_404(Project, project_id=project_id)
    
    #get contact information if available
    xbrws_project = Project.objects.get(project_id=project_id)
    seqr_project = xbrws_project.seqr_project 
    #Ideally these (the default Sam, and matchmaker@broad) should be set at project creation   
    seqr_project.is_mme_enabled=True 
    seqr_project.mme_primary_data_owner=settings.MME_CONTACT_NAME
    seqr_project.mme_contact_url=settings.MME_CONTACT_HREF
    
    #species (only human for now) till seqr starts tracking species
    species="NCBITaxon:9606"
    
    contact={
             "name":seqr_project.mme_primary_data_owner,
             "institution" : settings.MME_CONTACT_INSTITUTION,
             "href" : seqr_project.mme_contact_url
             }
        
    #genomicFeatures section
    genomic_features=[]
    variants=[]
    project_tags = ProjectTag.objects.filter(project__project_id=project_id)
    for project_tag in project_tags:
        variant_tags = VariantTag.objects.filter(project_tag=project_tag)
        for variant_tag in variant_tags:    
            if family_id == variant_tag.toJSON()['family']:
                variant = get_datastore(project.project_id).get_single_variant(
                        project.project_id,
                        variant_tag.toJSON()['family'],
                        variant_tag.xpos,
                        variant_tag.ref,
                        variant_tag.alt,
                )
                if variant is None:
                    logging.info("Variant no longer called in this family (did the callset version change?)")
                    continue
                variants.append({"variant":variant.toJSON(),
                                 "tag":project_tag.title,
                                 "family":variant_tag.family.toJSON(),
                                 "tag_name":variant_tag.toJSON()['tag']
                                 })
                
    current_genome_assembly = find_genome_assembly(seqr_project)
    #start compiling a matchmaker-esque data structure to send back
    genomic_features=[]
    for variant in variants:
        start = variant['variant']['pos']
        reference_bases = variant['variant']['ref']
        alternate_bases = variant['variant']['alt']
        end = int(variant['variant']['pos_end']) #int and long are unified in python
        reference_name = variant['variant']['chr'].replace('chr','')        
        #now we have more than 1 gene associated to these VAR postions,
        #so we will associate that information to each gene symbol
        for i,gene_id in enumerate(variant['variant']['gene_ids']):
            genomic_feature = {}
            genomic_feature['gene'] ={"id": gene_id }
            genomic_feature['variant']={
                                        'assembly':current_genome_assembly,
                                        'referenceBases':reference_bases,
                                        'alternateBases':alternate_bases,
                                        'start':start,
                                        'end':end,
                                        'referenceName':reference_name
                                        }
            genomic_feature['zygosity'] = variant['variant']['genotypes'][indiv_id]['num_alt']
            gene_symbol=""
            if gene_id != "":
                gene = get_reference().get_gene(gene_id)
                gene_symbol = gene['symbol']

            genomic_feature['auxiliary']={
                                          "tag_name":variant['tag_name'],
                                          "gene_symbol":gene_symbol
                                          }
            genomic_features.append(genomic_feature) 
    
    #Find phenotype information
    indiv = Individual.objects.get(indiv_id=indiv_id,project=project)
    phenotypes_entered = get_phenotypes_entered_for_individual(project_id,indiv.phenotips_id)
    #need to eventually support "FEMALE"|"MALE"|"OTHER"|"MIXED_SAMPLE"|"NOT_APPLICABLE",
    #as of now PhenoTips only has M/F
    sex="NOT_APPLICABLE"
    if "M" == indiv.gender:
        sex="MALE"
    if "F" == indiv.gender:
        sex="FEMALE"
    features=[]
    if phenotypes_entered.has_key('features'):
        #as of now non-standard features ('nonstandard_features') without HPO
        #terms cannot be sent to MME
        for f in phenotypes_entered['features']:
            features.append({
                "id":f['id'],
                "observed":f['observed'],
                "label":f['label']})
    
    id=indiv.indiv_id
    label=indiv.indiv_id

    #add new patient to affected patients
    affected_patient={"id":id,
                      "species":species,
                      "label":label,
                      "contact":contact,
                      "features":features,
                      "sex":sex,
                      "genomicFeatures":genomic_features
                      }
                            
    #map to put into mongo
    time_stamp = datetime.datetime.fromtimestamp(time.time()).strftime('%Y_%m_%d_%H_%M_%S')
    detailed_id_map={"generated_on": time_stamp,
         "project_id":project_id,
         "family_id":family_id,
         "individual_id":indiv.indiv_id,
         "mme_id":id,
         "individuals_used_for_phenotypes":affected_patient}
    return detailed_id_map,affected_patient
            
            
def find_genome_assembly(project):
    """
    Find the genome assembly of this individual
    Args:
        project: This is a seqr.project object reprenting the project
    Returns:
    The genome assembly version
    """
    datasets =  Dataset.objects.filter(analysis_type=Dataset.ANALYSIS_TYPE_VARIANT_CALLS, project=project)
    if datasets and datasets[0].genome_version:
        return 'GRCh' + datasets[0].genome_version
    return 'GRCh37'

def is_a_valid_patient_structure(patient_struct):
    """
    Checks to see if the input patient data structure has all the
    data/fields required by the MME
    Args:
        patient structure
    Returns:
        True if valid
    TODO:
        This function needs improvement and checks on field naming and other required values.
    """
    submission_validity={"status":True, "reason":""}
    #check if all gene IDs are present
{'fraction_non_alphanumeric': 0.08479760510217363, 'fraction_numerical': 0.001757126122608356, 'mean_word_length': 2.217546063651591, 'pattern_counts': {'":': 56, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 30, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '116355', 'n_tokens_mistral': 4357, 'n_tokens_neox': 4118, 'n_words': 925}