<commit_before>from django.utils.text import slugify
from django.db import IntegrityError
from protein.models import Protein, ProteinConformation
from residue.models import Residue
from structure.models import Structure
from construct.models import *

from ligand.models import Ligand, LigandType, LigandRole
from ligand.functions import get_or_make_ligand

from common.tools import fetch_from_web_api
from urllib.parse import quote
from string import Template
from urllib.request import urlopen
from operator import itemgetter
from itertools import groupby
import time
import json
import datetime
from collections import OrderedDict
import pickle
import logging

AA_three = {'CYS': 'C', 'ASP': 'D', 'SER': 'S', 'GLN': 'Q', 'LYS': 'K',
     'ILE': 'I', 'PRO': 'P', 'THR': 'T', 'PHE': 'F', 'ASN': 'N', 
     'GLY': 'G', 'HIS': 'H', 'LEU': 'L', 'ARG': 'R', 'TRP': 'W', 
     'ALA': 'A', 'VAL':'V', 'GLU': 'E', 'TYR': 'Y', 'MET': 'M'}

# def look_for_value(d,k):
#     ### look for a value in dict if found, give back, otherwise None


def fetch_pdb_info(pdbname,protein):
    SIFT_exceptions = {'5GLI':[395,403], '5GLH':[395,401]}
    logger = logging.getLogger('build')
    #d = {}
    d = OrderedDict()
    d['construct_crystal'] = {}
    d['construct_crystal']['pdb'] = pdbname
    d['construct_crystal']['pdb_name'] = 'auto_'+pdbname
    try:
        d['construct_crystal']['uniprot'] = protein.parent.entry_name
        d['protein'] = protein.parent.name
        d['wt_seq'] = protein.parent.sequence
    except:
        # if above fails, it's likely due to there being no child, and we are looking at top level
        d['construct_crystal']['uniprot'] = protein.entry_name
        d['protein'] = protein.name
        d['wt_seq'] = protein.sequence


    d['contact_info'] = {}
    d['contact_info']['name_cont'] = 'gpcrdb'
    d['contact_info']['pi_email'] = '<EMAIL>'
    d['contact_info']['pi_name'] = 'gpcrdb'
    d['contact_info']['url'] = 'gpcrdb.org'
    d['contact_info']['date'] = time.strftime('%m/%d/%Y')
    d['contact_info']['address'] = ''

    d['pdb'] = pdbname
    d['links'] = []
    d['xml_not_observed'] = []
    d['xml_segments'] = []

    pos_in_wt = list(range(1,len(d['wt_seq'])+1))

    #http://files.gpcrdb.org/uniprot_mapping.txt
    ## get uniprot to name mapping
    url = 'http://files.gpcrdb.org/uniprot_mapping.txt'
    req = urlopen(url)
    uniprot_mapping = req.read().decode('UTF-8')
    rows = ( line.split(' ') for line in uniprot_mapping.split('\n') )
    uniprot_mapping = { row[0]:row[1:] for row in rows }

    #errors, fix it.
    uniprot_mapping['P08483'] = ['acm3_rat']
    uniprot_mapping['P42866'] = ['oprm_mouse']
     
    variants_mapping = {}
    cache_dir = ['sifts', 'xml']
    url = 'http://www.uniprot.org/uniprot/$index.xml'
    insert_info = fetch_from_web_api(url, d['construct_crystal']['uniprot'], cache_dir, xml = True)
    for elm in insert_info.findall('.//{http://uniprot.org/uniprot}feature'):
        if elm.attrib['type']=="sequence variant":
            if 'description' in elm.attrib:
                desc = elm.attrib['description']
            else:
                desc = ''
            if 'id' in elm.attrib:
                var_id = elm.attrib['id']
            else :
                var_id = None
           #  print(desc,var_id)
            try:
                ori = elm.find('{http://uniprot.org/uniprot}original').text
                var = elm.find('{http://uniprot.org/uniprot}variation').text
                pos = elm.find('{http://uniprot.org/uniprot}location')[0].attrib['position']
                if pos not in variants_mapping:
                    variants_mapping[pos] = {}
                if var not in variants_mapping[pos]:
                    variants_mapping[pos][var] = []
                variants_mapping[pos][var].append([desc,var_id])
            except:
                pass
    for elm in insert_info.findall('.//{http://uniprot.org/uniprot}sequence'):
        uniprot_seq = elm.text
    # print(variants_mapping)
    # print("gpcrdb seq",len(d['wt_seq']),'uniport len',len(uniprot_seq))
    #ftp://ftp.ebi.ac.uk/pub/databases/msd/sifts/xml/1xyz.xml.gz
    url = 'ftp://ftp.ebi.ac.uk/pub/databases/msd/sifts/xml/$index.xml.gz'
    sifts = fetch_from_web_api(url, pdbname.lower(), cache_dir, xml = True)
    d['links'].append(Template(url).substitute(index=quote(str(pdbname.lower()), safe='')))
    d['mutations'] = []
    d['auxiliary'] = OrderedDict()
    d['construct_sequences'] = OrderedDict()
    receptor_seq_ids = []
    other_seq_ids = {}
    receptor_chain = ''
    uniprot_to_name = {}
    mutations_check = []
    if sifts: #success
        insert_position = 'N-term'<commit_msg>Add alternative link to pdb sifts in notes<commit_after>from django.utils.text import slugify
from django.db import IntegrityError
from protein.models import Protein, ProteinConformation
from residue.models import Residue
from structure.models import Structure
from construct.models import *

from ligand.models import Ligand, LigandType, LigandRole
from ligand.functions import get_or_make_ligand

from common.tools import fetch_from_web_api
from urllib.parse import quote
from string import Template
from urllib.request import urlopen
from operator import itemgetter
from itertools import groupby
import time
import json
import datetime
from collections import OrderedDict
import pickle
import logging

AA_three = {'CYS': 'C', 'ASP': 'D', 'SER': 'S', 'GLN': 'Q', 'LYS': 'K',
     'ILE': 'I', 'PRO': 'P', 'THR': 'T', 'PHE': 'F', 'ASN': 'N', 
     'GLY': 'G', 'HIS': 'H', 'LEU': 'L', 'ARG': 'R', 'TRP': 'W', 
     'ALA': 'A', 'VAL':'V', 'GLU': 'E', 'TYR': 'Y', 'MET': 'M'}

# def look_for_value(d,k):
#     ### look for a value in dict if found, give back, otherwise None


def fetch_pdb_info(pdbname,protein):
    SIFT_exceptions = {'5GLI':[395,403], '5GLH':[395,401]}
    logger = logging.getLogger('build')
    #d = {}
    d = OrderedDict()
    d['construct_crystal'] = {}
    d['construct_crystal']['pdb'] = pdbname
    d['construct_crystal']['pdb_name'] = 'auto_'+pdbname
    try:
        d['construct_crystal']['uniprot'] = protein.parent.entry_name
        d['protein'] = protein.parent.name
        d['wt_seq'] = protein.parent.sequence
    except:
        # if above fails, it's likely due to there being no child, and we are looking at top level
        d['construct_crystal']['uniprot'] = protein.entry_name
        d['protein'] = protein.name
        d['wt_seq'] = protein.sequence


    d['contact_info'] = {}
    d['contact_info']['name_cont'] = 'gpcrdb'
    d['contact_info']['pi_email'] = '<EMAIL>'
    d['contact_info']['pi_name'] = 'gpcrdb'
    d['contact_info']['url'] = 'gpcrdb.org'
    d['contact_info']['date'] = time.strftime('%m/%d/%Y')
    d['contact_info']['address'] = ''

    d['pdb'] = pdbname
    d['links'] = []
    d['xml_not_observed'] = []
    d['xml_segments'] = []

    pos_in_wt = list(range(1,len(d['wt_seq'])+1))

    #http://files.gpcrdb.org/uniprot_mapping.txt
    ## get uniprot to name mapping
    url = 'http://files.gpcrdb.org/uniprot_mapping.txt'
    req = urlopen(url)
    uniprot_mapping = req.read().decode('UTF-8')
    rows = ( line.split(' ') for line in uniprot_mapping.split('\n') )
    uniprot_mapping = { row[0]:row[1:] for row in rows }

    #errors, fix it.
    uniprot_mapping['P08483'] = ['acm3_rat']
    uniprot_mapping['P42866'] = ['oprm_mouse']
     
    variants_mapping = {}
    cache_dir = ['sifts', 'xml']
    url = 'http://www.uniprot.org/uniprot/$index.xml'
    insert_info = fetch_from_web_api(url, d['construct_crystal']['uniprot'], cache_dir, xml = True)
    for elm in insert_info.findall('.//{http://uniprot.org/uniprot}feature'):
        if elm.attrib['type']=="sequence variant":
            if 'description' in elm.attrib:
                desc = elm.attrib['description']
            else:
                desc = ''
            if 'id' in elm.attrib:
                var_id = elm.attrib['id']
            else :
                var_id = None
           #  print(desc,var_id)
            try:
                ori = elm.find('{http://uniprot.org/uniprot}original').text
                var = elm.find('{http://uniprot.org/uniprot}variation').text
                pos = elm.find('{http://uniprot.org/uniprot}location')[0].attrib['position']
                if pos not in variants_mapping:
                    variants_mapping[pos] = {}
                if var not in variants_mapping[pos]:
                    variants_mapping[pos][var] = []
                variants_mapping[pos][var].append([desc,var_id])
            except:
                pass
    for elm in insert_info.findall('.//{http://uniprot.org/uniprot}sequence'):
        uniprot_seq = elm.text
    # print(variants_mapping)
    # print("gpcrdb seq",len(d['wt_seq']),'uniport len',len(uniprot_seq))
    #ftp://ftp.ebi.ac.uk/pub/databases/msd/sifts/xml/1xyz.xml.gz
    # Alternative : url = 'http://www.rcsb.org/pdb/files/$index.sifts.xml'
    url = 'ftp://ftp.ebi.ac.uk/pub/databases/msd/sifts/xml/$index.xml.gz'
    sifts = fetch_from_web_api(url, pdbname.lower(), cache_dir, xml = True)
    d['links'].append(Template(url).substitute(index=quote(str(pdbname.lower()), safe='')))
    d['mutations'] = []
    d['auxiliary'] = OrderedDict()
    d['construct_sequences'] = OrderedDict()
    receptor_seq_ids = []
    other_seq_ids = {}
    receptor_chain = ''
    uniprot_to_name = {}
    mutations_check = []
    if sifts: #success
        insert_position = 'N-term'
{'fraction_non_alphanumeric': 0.16680707666385847, 'fraction_numerical': 0.006739679865206402, 'mean_word_length': 3.046442266723477, 'pattern_counts': {'":': 2, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 3, 'xml': 21}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2811736', 'n_tokens_mistral': 3426, 'n_tokens_neox': 3282, 'n_words': 645}