<commit_before>import logging
import re
import json
import urllib

from dipper.sources.Source import Source
from dipper.models.Dataset import Dataset
from dipper.models.Model import Model
from dipper.models.assoc.G2PAssoc import G2PAssoc
from dipper.models.Genotype import Genotype
from dipper.models.GenomicFeature import Feature, makeChromID
from dipper.models.Reference import Reference
from dipper import config
from dipper.utils.romanplus import romanNumeralPattern, fromRoman, toRoman

logger = logging.getLogger(__name__)


# omimftp key EXPIRES MAY 2018
# get a new one here: http://omim.org/help/api


OMIMFTP = 'http://data.omim.org/downloads/' + \
    config.get_config()['keys']['omim']

OMIMAPI = 'http://api.omim.org/api/entry?format=json&apiKey=' + \
    config.get_config()['keys']['omim'] + '&'


class OMIM(Source):
    """
    OMIM is an unusual source.
    The only anonymously obtainable data from the ftp site is mim2gene.
    However, more detailed information is available via their API.
    So, we pull the omim identifiers from their ftp site,
    then query their API in batchs of 20.
    Their prescribed rate limits have been mecurial
     one per two seconds or  four per second,
     in  2017 November all mention of api rate limits have vanished
     (save 20 IDs per call if any include is used)

    Note this ingest requires an apiKey which is not stored in the repo,
    but in a separate conf.json file.

    Processing this source serves two purposes:
    1.  the creation of the OMIM classes for merging into the disease ontology
    2.  add annotations such as disease-gene associations

    When creating the disease classes, we pull from their REST-api
    id/label/definition information.
    Additionally we pull the Orphanet and UMLS mappings
    (to make equivalent ids).
    We also pull the phenotypic series annotations as grouping classes.


    """

    files = {
        'all': {
            'file': 'mim2gene.txt',
            'url': 'http://omim.org/static/omim/data/mim2gene.txt'},
        'morbidmap': {
            'file': 'morbidmap.txt',
            'url':  OMIMFTP + '/morbidmap.txt'},
        'phenotypicSeries': {
            'file': 'phenotypic_series_title_all.txt',
            'url': 'http://www.omim.org/phenotypicSeriesTitle/all?format=tsv',
            'headers': {'User-Agent': 'Mozilla/5.0'}}

        # FTP files
        # /mimTitles.txt
        # /genemap.txt
        # /genemap2.txt
        # /omim.txt.Z                   replaced by mim2gene.txt
        # Download from a page not FTP
        # /phenotypic_series_title_all.txt
    }

    # the following test ids are in the config.json
    # (they do not belong there either  -TEC)
    test_ids = [
        119600, 120160, 157140, 158900, 166220, 168600, 219700,
        # coriell
        253250, 305900, 600669, 601278, 602421, 605073, 607822,
        # genes
        102560, 102480, 100678, 102750, 600201,
        # phenotype/disease -- indicate that here?
        104200, 105400, 114480, 115300, 121900,
        # gene of known sequence and has a phenotype
        107670, 11600, 126453,
        102150, 104000, 107200, 100070, 611742, 611100,
        # disease with known locus
        102480]<commit_msg>comments, add a 'clean' url for the dataset file<commit_after>import logging
import re
import json
import urllib

from dipper.sources.Source import Source
from dipper.models.Dataset import Dataset
from dipper.models.Model import Model
from dipper.models.assoc.G2PAssoc import G2PAssoc
from dipper.models.Genotype import Genotype
from dipper.models.GenomicFeature import Feature, makeChromID
from dipper.models.Reference import Reference
from dipper import config
from dipper.utils.romanplus import romanNumeralPattern, fromRoman, toRoman

logger = logging.getLogger(__name__)


# omimftp key EXPIRES MAY 2018
# get a new one here: https://omim.org/help/api

OMIMURL = 'https://data.omim.org/downloads/'

OMIMFTP = OMIMURL + config.get_config()['keys']['omim']

OMIMAPI = 'https://api.omim.org/api/entry?format=json&apiKey=' + \
    config.get_config()['keys']['omim'] + '&'


class OMIM(Source):
    """
    OMIM is an unusual source.
    The only anonymously obtainable data from the ftp site is mim2gene.
    However, more detailed information is available via their API.
    So, we pull the omim identifiers from their ftp site,
    then query their API in batchs of 20.
    Their prescribed rate limits have been mecurial
     one per two seconds or  four per second,
     in  2017 November all mention of api rate limits have vanished
     (save 20 IDs per call if any include is used)

    Note this ingest requires an api Key which is not stored in the repo,
    but in a separate conf.json file.

    Processing this source serves two purposes:
    1.  the creation of the OMIM classes for merging into the disease ontology
    2.  add annotations such as disease-gene associations

    When creating the disease classes, we pull from their REST-api
    id/label/definition information.
    Additionally we pull the Orphanet and UMLS mappings
    (to make equivalent ids).
    We also pull the phenotypic series annotations as grouping classes.


    """

    files = {
        'all': {
            'file': 'mim2gene.txt',
            'url': 'http://omim.org/static/omim/data/mim2gene.txt'},
        'morbidmap': {
            'file': 'morbidmap.txt',
            'url':  OMIMFTP + '/morbidmap.txt',
            'clean': OMIMURL},
        'phenotypicSeries': {
            'file': 'phenotypic_series_title_all.txt',
            'url': 'http://www.omim.org/phenotypicSeriesTitle/all?format=tsv',
            'headers': {'User-Agent': 'Mozilla/5.0'}}

        # FTP files
        # /mimTitles.txt
        # /genemap.txt
        # /genemap2.txt
        # /omim.txt.Z                   replaced by mim2gene.txt
        # Download from a page not FTP
        # /phenotypic_series_title_all.txt
    }

    # the following test ids are in the config.json
    # (they do not belong there either  -TEC)
    test_ids = [
        119600, 120160, 157140, 158900, 166220, 168600, 219700,
        # coriell
        253250, 305900, 600669, 601278, 602421, 605073, 607822,
        # genes
        102560, 102480, 100678, 102750, 600201,
        # phenotype/disease -- indicate that here?
        104200, 105400, 114480, 115300, 121900,
        # gene of known sequence and has a phenotype
        107670, 11600, 126453,
        102150, 104000, 107200, 100070, 611742, 611100,
        # disease with known locus
        102480]
{'fraction_non_alphanumeric': 0.08832951945080092, 'fraction_numerical': 0.06895499618611747, 'mean_word_length': 2.701863354037267, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 3, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6473893', 'n_tokens_mistral': 2407, 'n_tokens_neox': 1979, 'n_words': 672}