<commit_before>import csv
import os
from datetime import datetime
from stat import *
import urllib
import logging

from dipper.utils import pysed
from dipper.utils.GraphUtils import GraphUtils
from dipper.sources.Source import Source
from dipper.models.D2PAssoc import D2PAssoc
from dipper.models.DispositionAssoc import DispositionAssoc
from dipper.models.Dataset import Dataset
from dipper.models.Assoc import Assoc
from dipper import curie_map
from dipper import config


'''
#see info on format here:http://www.human-phenotype-ontology.org/contao/index.php/annotation-guide.html
#file schema:
#1 	DB 	required 	MIM
#2 	DB_Object_ID 	required 	154700
#3 	DB_Name 	required 	Achondrogenesis, type IB
#4 	Qualifier 	optional 	NOT
#5 	HPO ID 	required 	HP:0002487
#6 	DB:Reference 	required 	OMIM:154700 or PMID:15517394
#7 	Evidence code 	required 	IEA
#8 	Onset modifier  optional	HP:0003577
#9	Frequency modifier	optional	"70%" or "12 of 30" or from the vocabulary show in table 2
#10 With 	optional
#11 Aspect 	required 	O
#12 	Synonym 	optional 	ACG1B|Achondrogenesis, Fraccaro type
#13 	Date 	required 	YYYY.MM.DD
#14 	Assigned by 	required 	HPO
'''

logger = logging.getLogger(__name__)


class HPOAnnotations(Source):

    files = {
        'annot': {'file' : 'phenotype_annotation.tab',
                   'url' : 'http://compbio.charite.de/hudson/job/hpo.annotations/lastStableBuild/artifact/misc/phenotype_annotation.tab'},
#       'neg_annot': {'file' : 'phenotype_annotation.tab',
#                     'url' : 'http://compbio.charite.de/hudson/job/hpo.annotations/lastStableBuild/artifact/misc/negative_phenotype_annotation.tab'
#        },
    }

    # note, two of these codes are awaiting term requests
    # https://code.google.com/p/evidenceontology/issues/detail?id=32
    eco_dict = {
        "ICE": "ECO:0000305",  # FIXME currently using "curator inference used in manual assertion"
        "IEA": "ECO:0000501",  # Inferred from Electronic Annotation
        "PCS": "ECO:0000269",  # FIXME currently using "experimental evidence used in manual assertion"
        "TAS": "ECO:0000304"   # Traceable Author Statement
    }

    def __init__(self):
        Source.__init__(self, 'hpoa')

        self.load_bindings()

        self.dataset = Dataset('hpoa', 'Human Phenotype Ontology',
                               'http://www.human-phenotype-ontology.org', None,
                               'http://www.human-phenotype-ontology.org/contao/index.php/legal-issues.html')

        if 'test_ids' not in config.get_config() and 'disease' not in config.get_config()['test_ids']:
            logger.warn("not configured with gene test ids.")
        else:
            self.test_ids = config.get_config()['test_ids']['disease']

        # data-source specific warnings (will be removed when issues are cleared)
        logger.warn("note that some ECO classes are missing for ICE and PCS; using temporary mappings.")

        return

    def fetch(self, is_dl_forced):
        st = None
        for f in self.files.keys():
            file = self.files.get(f)
            self.fetch_from_url(file['url'],
                                '/'.join((self.rawdir, file['file'])),
                                is_dl_forced)
            self.dataset.setFileAccessUrl(file['url'])
            # zfin versions are set by the date of download.
            st = os.stat('/'.join((self.rawdir, file['file'])))

        filedate = datetime.utcfromtimestamp(st[ST_CTIME]).strftime("%Y-%m-%d")

        self.scrub()

        # get the latest build from jenkins
        jenkins_info = eval(urllib.request.urlopen('http://compbio.charite.de/hudson/job/hpo.annotations/lastSuccessfulBuild/api/python').read())
        version = jenkins_info['number']

        self.dataset.setVersion(filedate, str(version))

        return

    def scrub(self):
        """
        Perform various data-scrubbing on the raw data files prior to parsing.
        For this resource, this currently includes:
        * revise errors in identifiers for some OMIM and PMIDs
        :return: None
        """
        # scrub file of the oddities...lots of publication rewriting
        f = '/'.join((self.rawdir, self.files['annot']['file']))
        logger.info('scrubbing PubMed:12345 --> PMID:12345')
        pysed.replace("PubMed", 'PMID', f)

        logger.info('scrubbing pmid:12345 --> PMID:12345')
        pysed.replace("pmid", 'PMID', f)

        logger.info('scrubbing PMID12345 --> PMID:12345')
        pysed.replace("PMID([0-9][0-9]*)", 'PMID:\\1', f)

        logger.info('scrubbing MIM12345 --> OMIM:12345')<commit_msg>using date_version from repo #16<commit_after>import csv
import os
from datetime import datetime
from stat import *
import urllib
import logging

from dipper.utils import pysed
from dipper.utils.GraphUtils import GraphUtils
from dipper.sources.Source import Source
from dipper.models.D2PAssoc import D2PAssoc
from dipper.models.DispositionAssoc import DispositionAssoc
from dipper.models.Dataset import Dataset
from dipper.models.Assoc import Assoc
from dipper import curie_map
from dipper import config


'''
#see info on format here:http://www.human-phenotype-ontology.org/contao/index.php/annotation-guide.html
#file schema:
#1 	DB 	required 	MIM
#2 	DB_Object_ID 	required 	154700
#3 	DB_Name 	required 	Achondrogenesis, type IB
#4 	Qualifier 	optional 	NOT
#5 	HPO ID 	required 	HP:0002487
#6 	DB:Reference 	required 	OMIM:154700 or PMID:15517394
#7 	Evidence code 	required 	IEA
#8 	Onset modifier  optional	HP:0003577
#9	Frequency modifier	optional	"70%" or "12 of 30" or from the vocabulary show in table 2
#10 With 	optional
#11 Aspect 	required 	O
#12 	Synonym 	optional 	ACG1B|Achondrogenesis, Fraccaro type
#13 	Date 	required 	YYYY.MM.DD
#14 	Assigned by 	required 	HPO
'''

logger = logging.getLogger(__name__)


class HPOAnnotations(Source):

    files = {
        'annot': {'file' : 'phenotype_annotation.tab',
                   'url' : 'http://compbio.charite.de/hudson/job/hpo.annotations/lastStableBuild/artifact/misc/phenotype_annotation.tab'},
        'version': {'file' : 'data_version.txt',
                   'url' : 'http://compbio.charite.de/hudson/job/hpo.annotations/lastStableBuild/artifact/misc/data_version.txt'},
#       'neg_annot': {'file' : 'phenotype_annotation.tab',
#                     'url' : 'http://compbio.charite.de/hudson/job/hpo.annotations/lastStableBuild/artifact/misc/negative_phenotype_annotation.tab'
#        },
    }

    # note, two of these codes are awaiting term requests
    # https://code.google.com/p/evidenceontology/issues/detail?id=32
    eco_dict = {
        "ICE": "ECO:0000305",  # FIXME currently using "curator inference used in manual assertion"
        "IEA": "ECO:0000501",  # Inferred from Electronic Annotation
        "PCS": "ECO:0000269",  # FIXME currently using "experimental evidence used in manual assertion"
        "TAS": "ECO:0000304"   # Traceable Author Statement
    }

    def __init__(self):
        Source.__init__(self, 'hpoa')

        self.load_bindings()

        self.dataset = Dataset('hpoa', 'Human Phenotype Ontology',
                               'http://www.human-phenotype-ontology.org', None,
                               'http://www.human-phenotype-ontology.org/contao/index.php/legal-issues.html')

        if 'test_ids' not in config.get_config() and 'disease' not in config.get_config()['test_ids']:
            logger.warn("not configured with gene test ids.")
        else:
            self.test_ids = config.get_config()['test_ids']['disease']

        # data-source specific warnings (will be removed when issues are cleared)
        logger.warn("note that some ECO classes are missing for ICE and PCS; using temporary mappings.")

        return

    def fetch(self, is_dl_forced):

        self.get_files(is_dl_forced)

        # st = None
        # for f in self.files.keys():
        #     file = self.files.get(f)
        #     self.fetch_from_url(file['url'],
        #                         '/'.join((self.rawdir, file['file'])),
        #                         is_dl_forced)
        #     self.dataset.setFileAccessUrl(file['url'])
        #     # zfin versions are set by the date of download.
        #     st = os.stat('/'.join((self.rawdir, file['file'])))
        #
        # filedate = datetime.utcfromtimestamp(st[ST_CTIME]).strftime("%Y-%m-%d")

        self.scrub()

        # get the latest build from jenkins
        # NOT DOING THIS ANY MORE - but leaving it in for reference
        # jenkins_info = eval(urllib.request.urlopen('http://compbio.charite.de/hudson/job/hpo.annotations/lastSuccessfulBuild/api/python').read())
        # version = jenkins_info['number']

        # use the files['version'] file as the version
        fname = '/'.join((self.rawdir, self.files['version']['file']))
        d = ""
        with open(fname, 'r', encoding="utf8") as f:
            # 2015-04-23 13:01
            v = f.readline()  # read the first line (the only line, really)
            d = datetime.strptime(v.strip(), '%Y-%m-%d %H:%M').strftime("%Y-%m-%d-%H-%M")
        f.close()

        st = os.stat(fname)
        filedate = datetime.utcfromtimestamp(st[ST_CTIME]).strftime("%Y-%m-%d")

        # this will cause two dates to be attached to the dataset (one from the filedate, and the other from here)
        # TODO when #112 is implemented, this will result in only the whole dataset being versioned
        self.dataset.setVersion(filedate, d)

        return

    def scrub(self):
        """
        Perform various data-scrubbing on the raw data files prior to parsing.
        For this resource, this currently includes:
        * revise errors in identifiers for some OMIM and PMIDs
        :return: None
        """
        # scrub file of the oddities...lots of publication rewriting
        f = '/'.join((self.rawdir, self.files['annot']['file']))
        logger.info('scrubbing PubMed:12345 --> PMID:12345')
        pysed.replace("PubMed", 'PMID', f)

        logger.info('scrubbing pmid:12345 --> PMID:12345')
        pysed.replace("pmid", 'PMID', f)

        logger.info('scrubbing PMID12345 --> PMID:12345')
        pysed.replace("PMID([0-9][0-9]*)", 'PMID:\\1', f)

        logger.info('scrubbing MIM12345 --> OMIM:12345')
{'fraction_non_alphanumeric': 0.11885325558794947, 'fraction_numerical': 0.02857142857142857, 'mean_word_length': 2.729974628488583, 'pattern_counts': {'":': 8, '<': 3, '<?xml version=': 0, '>': 11, 'https://': 2, 'lorem ipsum': 0, 'www.': 6, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7198737', 'n_tokens_mistral': 3533, 'n_tokens_neox': 3216, 'n_words': 809}