<commit_before>"""
This script retrieves OMIM data from omim.org and parses/converts relevant fields into a tsv table.

==================
OMIM DATA SOURCES:
==================
OMIM provides data through an API (https://omim.org/help/api) and in downloadable files (https://omim.org/downloads/)
The geneMap API endpoint provides only gene symbols and not the Ensembl gene id, while
genemap2.txt provides both, so we use genemap2.txt as the data source.


API endpoints:
-------------
http://api.omim.org/api/geneMap?chromosome=1
   returns a list of 'geneMap' objects - each representing a
   mimNumber, geneSymbols, geneName, comments, geneInheritance, and a phenotypeMapList
   which contains one or more mimNumber, phenotypeMimNumber, phenotype description, and
   phenotypeInheritance

http://api.omim.org/api/entry?mimNumber=612367&format=json&include=all
   returns detailed info on a particular mim id

Files:
-----
mim2gene.txt - contains basic info on mim numbers and their relationships.

For example:
     100500  moved/removed
     100600  phenotype
     100640  gene    216     ALDH1A1 ENSG00000165092,ENST00000297785
     100650  gene/phenotype  217     ALDH2   ENSG00000111275,ENST00000261733

genemap2.txt - contains chrom, gene_start, gene_end, cyto_location, mim_number,
    gene_symbols, gene_name, approved_symbol, entrez_gene_id, ensembl_gene_id, comments, phenotypes,
    mouse_gene_id  -  where phenotypes contains 1 or more phenotypes in the form
    { description }, phenotype_mim_number (phenotype_mapping_key), inheritance_mode;

Example genemap2.txt record:

   # Chromosome    Genomic Position Start    Genomic Position End    Cyto Location    Computed Cyto Location    Mim Number    Gene Symbols    Gene Name    Approved Symbol    Entrez Gene ID    Ensembl Gene ID    Comments    Phenotypes    Mouse Gene Symbol/ID
   chr1    2019328    2030752    1p36.33        137163    GABRD, GEFSP5, EIG10, EJM7    Gamma-aminobutyric acid (GABA) A receptor, delta    GABRD    2563    ENSG00000187730        {Epilepsy, generalized, with febrile seizures plus, type 5, susceptibility to}, 613060 (3), Autosomal dominant; {Epilepsy, idiopathic generalized, 10}, 613060 (3), Autosomal dominant; {Epilepsy, juvenile myoclonic, susceptibility to}, 613060 (3), Autosomal dominant    Gabrd (MGI:95622)

"""

import json
import logging
import os
import re
import requests

from django.core.management.base import CommandError

from reference_data.management.commands.utils.update_utils import GeneCommand, ReferenceDataHandler
from reference_data.models import Omim

logger = logging.getLogger(__name__)

OMIM_ENTRIES_URL = 'https://api.omim.org/api/entry?apiKey={omim_key}&include=geneMap&format=json&mimNumber={mim_numbers}'

OMIM_PHENOTYPE_MAP_METHOD_CHOICES = {
    1: 'the disorder is placed on the map based on its association with a gene, but the underlying defect is not known.',
    2: 'the disorder has been placed on the map by linkage; no mutation has been found.',
    3: 'the molecular basis for the disorder is known; a mutation has been found in the gene.',
    4: 'a contiguous gene deletion or duplication syndrome, multiple genes are deleted or duplicated causing the phenotype.',
}


class OmimReferenceDataHandler(ReferenceDataHandler):

    model_cls = Omim
    url = "http://data.omim.org/downloads/{omim_key}/genemap2.txt"

    def __init__(self, omim_key=None, **kwargs):
        if not omim_key:
            raise CommandError("omim_key is required")

        self.url = self.url.format(omim_key=omim_key)
        self.omim_key = omim_key
        super(OmimReferenceDataHandler, self).__init__()

    @staticmethod
    def get_file_header(f):
        header_fields = None
        for line in f:
            line = line.rstrip('\r\n')
            if line.startswith("# Chrom") and header_fields is None:
                header_fields = [c.lower().replace(' ', '_') for c in line.split('\t')]
                break
            elif not line or line.startswith("#"):
                continue
            elif line.startswith('This account is inactive'):
                raise Exception(line)
            elif header_fields is None:
                raise ValueError("Header row not found in genemap2 file before line {}: {}".format(i, line))

        return header_fields

    @staticmethod
    def parse_record(record):
        # skip commented rows
        if len(record) == 1:
            yield None

        else:
            # rename some of the fields
            output_record = {}
            output_record['gene_id'] = record['ensembl_gene_id']
            output_record['mim_number'] = int(record['mim_number'])
            output_record['gene_symbol'] = record['approved_symbol'].strip() or record['gene_symbols'].split(",")[0]
            output_record['gene_description'] = record['gene_name']
            output_record['comments'] = record['comments']<commit_msg>better error message when omim key expires<commit_after>"""
This script retrieves OMIM data from omim.org and parses/converts relevant fields into a tsv table.

==================
OMIM DATA SOURCES:
==================
OMIM provides data through an API (https://omim.org/help/api) and in downloadable files (https://omim.org/downloads/)
The geneMap API endpoint provides only gene symbols and not the Ensembl gene id, while
genemap2.txt provides both, so we use genemap2.txt as the data source.


API endpoints:
-------------
http://api.omim.org/api/geneMap?chromosome=1
   returns a list of 'geneMap' objects - each representing a
   mimNumber, geneSymbols, geneName, comments, geneInheritance, and a phenotypeMapList
   which contains one or more mimNumber, phenotypeMimNumber, phenotype description, and
   phenotypeInheritance

http://api.omim.org/api/entry?mimNumber=612367&format=json&include=all
   returns detailed info on a particular mim id

Files:
-----
mim2gene.txt - contains basic info on mim numbers and their relationships.

For example:
     100500  moved/removed
     100600  phenotype
     100640  gene    216     ALDH1A1 ENSG00000165092,ENST00000297785
     100650  gene/phenotype  217     ALDH2   ENSG00000111275,ENST00000261733

genemap2.txt - contains chrom, gene_start, gene_end, cyto_location, mim_number,
    gene_symbols, gene_name, approved_symbol, entrez_gene_id, ensembl_gene_id, comments, phenotypes,
    mouse_gene_id  -  where phenotypes contains 1 or more phenotypes in the form
    { description }, phenotype_mim_number (phenotype_mapping_key), inheritance_mode;

Example genemap2.txt record:

   # Chromosome    Genomic Position Start    Genomic Position End    Cyto Location    Computed Cyto Location    Mim Number    Gene Symbols    Gene Name    Approved Symbol    Entrez Gene ID    Ensembl Gene ID    Comments    Phenotypes    Mouse Gene Symbol/ID
   chr1    2019328    2030752    1p36.33        137163    GABRD, GEFSP5, EIG10, EJM7    Gamma-aminobutyric acid (GABA) A receptor, delta    GABRD    2563    ENSG00000187730        {Epilepsy, generalized, with febrile seizures plus, type 5, susceptibility to}, 613060 (3), Autosomal dominant; {Epilepsy, idiopathic generalized, 10}, 613060 (3), Autosomal dominant; {Epilepsy, juvenile myoclonic, susceptibility to}, 613060 (3), Autosomal dominant    Gabrd (MGI:95622)

"""

import json
import logging
import os
import re
import requests

from django.core.management.base import CommandError

from reference_data.management.commands.utils.update_utils import GeneCommand, ReferenceDataHandler
from reference_data.models import Omim

logger = logging.getLogger(__name__)

OMIM_ENTRIES_URL = 'https://api.omim.org/api/entry?apiKey={omim_key}&include=geneMap&format=json&mimNumber={mim_numbers}'

OMIM_PHENOTYPE_MAP_METHOD_CHOICES = {
    1: 'the disorder is placed on the map based on its association with a gene, but the underlying defect is not known.',
    2: 'the disorder has been placed on the map by linkage; no mutation has been found.',
    3: 'the molecular basis for the disorder is known; a mutation has been found in the gene.',
    4: 'a contiguous gene deletion or duplication syndrome, multiple genes are deleted or duplicated causing the phenotype.',
}


class OmimReferenceDataHandler(ReferenceDataHandler):

    model_cls = Omim
    url = "http://data.omim.org/downloads/{omim_key}/genemap2.txt"

    def __init__(self, omim_key=None, **kwargs):
        if not omim_key:
            raise CommandError("omim_key is required")

        self.url = self.url.format(omim_key=omim_key)
        self.omim_key = omim_key
        super(OmimReferenceDataHandler, self).__init__()

    @staticmethod
    def get_file_header(f):
        header_fields = None
        for i, line in enumerate(f):
            line = line.rstrip('\r\n')
            if line.startswith("# Chrom") and header_fields is None:
                header_fields = [c.lower().replace(' ', '_') for c in line.split('\t')]
                break
            elif not line or line.startswith("#"):
                continue
            elif line.startswith('This account is inactive'):
                raise Exception(line)
            elif header_fields is None:
                raise ValueError("Header row not found in genemap2 file before line {}: {}".format(i, line))

        return header_fields

    @staticmethod
    def parse_record(record):
        # skip commented rows
        if len(record) == 1:
            yield None

        else:
            # rename some of the fields
            output_record = {}
            output_record['gene_id'] = record['ensembl_gene_id']
            output_record['mim_number'] = int(record['mim_number'])
            output_record['gene_symbol'] = record['approved_symbol'].strip() or record['gene_symbols'].split(",")[0]
            output_record['gene_description'] = record['gene_name']
            output_record['comments'] = record['comments']
{'fraction_non_alphanumeric': 0.09179429962470839, 'fraction_numerical': 0.03489197687392231, 'mean_word_length': 3.3207712532865905, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 6, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 6, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '928532', 'n_tokens_mistral': 3197, 'n_tokens_neox': 2817, 'n_words': 900}