<commit_before>import csv
import os
import logging
import re
import shutil
import tarfile
import glob
from datetime import datetime
from stat import ST_CTIME

import requests
from dipper.sources.Source import Source
from dipper.models.assoc.D2PAssoc import D2PAssoc
from dipper.models.Reference import Reference
from dipper.models.Model import Model
from dipper import config

LOG = logging.getLogger(__name__)

# summer 2018 PR mentioned switching to the new format
PNR = 'http://compbio.charite.de/jenkins/job/hpo.annotations.2018'
HPOADL2 = PNR + '/lastSuccessfulBuild/artifact/misc_2018'

# Fall 2018 CM made a mondo version of common disease (but we decided to hold off now)
GITAPI = "https://api.github.com/repos/monarch-initiative"

ORPHANET = 'http://www.orpha.net/consor/cgi-bin/OC_Exp.php?lng=en&Expert='
CONF = config.get_config()


class HPOAnnotations(Source):
    """
    The [Human Phenotype Ontology](http://human-phenotype-ontology.org) group
    curates and assembles over 115,000 annotations to hereditary diseases
    using the HPO ontology. Here we create OBAN-style associations
    between diseases and phenotypic features, together with their evidence,
    and age of onset and frequency (if known).
    The parser currently only processes the "abnormal" annotations.
    Association to "remarkable normality" will be added in the near future.

    We create additional associations from text mining.  See info at
    http://pubmed-browser.human-phenotype-ontology.org/.

    Also, you can read about these annotations in
    [PMID:26119816](http://www.ncbi.nlm.nih.gov/pubmed/26119816).

    In order to properly test this class,
    you should have a resources/test_ids.yaml file configured with some test ids,
    in the structure of:
    # as examples.  put your favorite ids in the config.
    <pre>
    test_ids: {"disease" : ["OMIM:119600", "OMIM:120160"]}
    </pre>

    """

    files = {
        'hpoa': {
            'file': 'phenotype.hpoa',
            'url':  HPOADL2 + '/phenotype.hpoa',
            'columns': [
                'DatabaseID',
                'DiseaseName',
                'Qualifier',
                'HPO_ID',
                'Reference',
                'Evidence',
                'Onset',
                'Frequency',
                'Sex',
                'Modifier',
                'Aspect',
                'Biocuration'
            ]
        },
        'doid': {  # DOID plans to remain all about inconsistant left zero padding
            'file': 'doid.owl',
            'url': 'http://purl.obolibrary.org/obo/doid.owl'
        }
    }

    small_files = {
        # this is a placeholder for the columns in the "common-disease"
        # small file format which were added to the 'files' dict programaticly
        'columns': [
            'Disease ID',
            'Disease Name',
            'Gene ID',
            'Gene Name',
            'Genotype',
            'Gene Symbol(s)',
            'Phenotype ID',
            'Phenotype Name',
            'Age of Onset ID',
            'Age of Onset Name',
            'Evidence ID',
            'Evidence Name',
            'Frequency',
            'Sex ID',
            'Sex Name',
            'Negation ID',
            'Negation Name',
            'Description',
            'Pub',
            'Assigned by',
            'Date Created'
        ]
    }

    def __init__(self, graph_type, are_bnodes_skolemized, skip_stats=False):
        super().__init__(
            graph_type=graph_type,
            are_bnodes_skized=are_bnodes_skolemized,
            skip_stats=skip_stats,
            name='hpoa',
            ingest_title='Human Phenotype Ontology',
            ingest_url='https://hpo.jax.org/app/',
            ingest_logo='https://github.com/monarch-initiative/monarch-ui/blob/master/public/img/sources/source-hpo.png',
            license_url=None,
            data_rights='https://hpo.jax.org/app/license',
            # file_handle=None
        )
        self.dataset.set_citation('https://hpo.jax.org/app/citation')
        self.replaced_id_count = 0
        self.test_ids = self.all_test_ids['disease']

        return

    def fetch(self, is_dl_forced=False):
        # get the latest build from jenkins
        self.get_files(is_dl_forced)
        # and git repo
        # self.get_common_files()
        return

    def parse(self, limit=None):
        if limit is not None:
            LOG.info("Only parsing first %s rows", limit)

        # self.add_common_files_to_file_list()
        LOG.info("Parsing files...")
        if self.test_only:
            self.test_mode = True

        self._process_phenotype_hpoa(self.files['hpoa'], limit=limit)

        # TODO add negative phenotype statements #113
        # self._process_negative_phenotype_tab(self.rawfile,self.outfile,limit)

        # common disease-phenotype associations from text mining work
        # self.process_all_common_disease_files(limit)

        LOG.info("Finished parsing.")

        return

    def _process_phenotype_hpoa(self, file_info, limit=None):
        """
        see info on format here:
        http://www.human-phenotype-ontology.org/contao/index.php/annotation-guide.html

        :param raw:
        :param limit:
        :return:

        """
        src_key = 'hpoa'

        if self.test_mode:
            graph = self.testgraph
        else:
            graph = self.graph
        model = Model(graph)

        raw = '/'.join((self.rawdir, file_info['file']))

        filedate = datetime.utcfromtimestamp(
            os.stat(raw)[ST_CTIME]).strftime("%Y-%m-%d")

        # this will cause two dates to be attached to the dataset
        # (one from the filedate, and the other from here)
        # TODO when #112 is implemented,
        # this will result in only the whole dataset being versioned

        col = self.files[src_key]['columns']
        with open(raw, 'r', encoding="utf8") as tsvfile:
            reader = csv.reader(tsvfile, delimiter='\t', quotechar='\"')
            row = next(reader)  # drop Description
            row = str(next(reader))[9:19]
            LOG.info("Ingest from %s", row)
            date = datetime.strptime(
                row.strip(), '%Y-%m-%d').strftime("%Y-%m-%d-%H-%M")

            if file_info.get("url") is not None:
                self.dataset.set_ingest_source_file_version_date(
                    file_info.get("url"), date)

            row = next(reader)  # drop tracker url<commit_msg>Removed unused variable<commit_after>import csv
import os
import logging
import re
import shutil
import tarfile
import glob
from datetime import datetime
from stat import ST_CTIME

import requests
from dipper.sources.Source import Source
from dipper.models.assoc.D2PAssoc import D2PAssoc
from dipper.models.Reference import Reference
from dipper.models.Model import Model
from dipper import config

LOG = logging.getLogger(__name__)

# summer 2018 PR mentioned switching to the new format
PNR = 'http://compbio.charite.de/jenkins/job/hpo.annotations.2018'
HPOADL2 = PNR + '/lastSuccessfulBuild/artifact/misc_2018'

# Fall 2018 CM made a mondo version of common disease (but we decided to hold off now)
GITAPI = "https://api.github.com/repos/monarch-initiative"

ORPHANET = 'http://www.orpha.net/consor/cgi-bin/OC_Exp.php?lng=en&Expert='
CONF = config.get_config()


class HPOAnnotations(Source):
    """
    The [Human Phenotype Ontology](http://human-phenotype-ontology.org) group
    curates and assembles over 115,000 annotations to hereditary diseases
    using the HPO ontology. Here we create OBAN-style associations
    between diseases and phenotypic features, together with their evidence,
    and age of onset and frequency (if known).
    The parser currently only processes the "abnormal" annotations.
    Association to "remarkable normality" will be added in the near future.

    We create additional associations from text mining.  See info at
    http://pubmed-browser.human-phenotype-ontology.org/.

    Also, you can read about these annotations in
    [PMID:26119816](http://www.ncbi.nlm.nih.gov/pubmed/26119816).

    In order to properly test this class,
    you should have a resources/test_ids.yaml file configured with some test ids,
    in the structure of:
    # as examples.  put your favorite ids in the config.
    <pre>
    test_ids: {"disease" : ["OMIM:119600", "OMIM:120160"]}
    </pre>

    """

    files = {
        'hpoa': {
            'file': 'phenotype.hpoa',
            'url':  HPOADL2 + '/phenotype.hpoa',
            'columns': [
                'DatabaseID',
                'DiseaseName',
                'Qualifier',
                'HPO_ID',
                'Reference',
                'Evidence',
                'Onset',
                'Frequency',
                'Sex',
                'Modifier',
                'Aspect',
                'Biocuration'
            ]
        },
        'doid': {  # DOID plans to remain all about inconsistant left zero padding
            'file': 'doid.owl',
            'url': 'http://purl.obolibrary.org/obo/doid.owl'
        }
    }

    small_files = {
        # this is a placeholder for the columns in the "common-disease"
        # small file format which were added to the 'files' dict programaticly
        'columns': [
            'Disease ID',
            'Disease Name',
            'Gene ID',
            'Gene Name',
            'Genotype',
            'Gene Symbol(s)',
            'Phenotype ID',
            'Phenotype Name',
            'Age of Onset ID',
            'Age of Onset Name',
            'Evidence ID',
            'Evidence Name',
            'Frequency',
            'Sex ID',
            'Sex Name',
            'Negation ID',
            'Negation Name',
            'Description',
            'Pub',
            'Assigned by',
            'Date Created'
        ]
    }

    def __init__(self, graph_type, are_bnodes_skolemized, skip_stats=False):
        super().__init__(
            graph_type=graph_type,
            are_bnodes_skized=are_bnodes_skolemized,
            skip_stats=skip_stats,
            name='hpoa',
            ingest_title='Human Phenotype Ontology',
            ingest_url='https://hpo.jax.org/app/',
            ingest_logo='https://github.com/monarch-initiative/monarch-ui/blob/master/public/img/sources/source-hpo.png',
            license_url=None,
            data_rights='https://hpo.jax.org/app/license',
            # file_handle=None
        )
        self.dataset.set_citation('https://hpo.jax.org/app/citation')
        self.replaced_id_count = 0
        self.test_ids = self.all_test_ids['disease']

        return

    def fetch(self, is_dl_forced=False):
        # get the latest build from jenkins
        self.get_files(is_dl_forced)
        # and git repo
        # self.get_common_files()
        return

    def parse(self, limit=None):
        if limit is not None:
            LOG.info("Only parsing first %s rows", limit)

        # self.add_common_files_to_file_list()
        LOG.info("Parsing files...")
        if self.test_only:
            self.test_mode = True

        self._process_phenotype_hpoa(self.files['hpoa'], limit=limit)

        # TODO add negative phenotype statements #113
        # self._process_negative_phenotype_tab(self.rawfile,self.outfile,limit)

        # common disease-phenotype associations from text mining work
        # self.process_all_common_disease_files(limit)

        LOG.info("Finished parsing.")

        return

    def _process_phenotype_hpoa(self, file_info, limit=None):
        """
        see info on format here:
        http://www.human-phenotype-ontology.org/contao/index.php/annotation-guide.html

        :param raw:
        :param limit:
        :return:

        """
        src_key = 'hpoa'

        if self.test_mode:
            graph = self.testgraph
        else:
            graph = self.graph
        model = Model(graph)

        raw = '/'.join((self.rawdir, file_info['file']))

        # this will cause two dates to be attached to the dataset
        # (one from the filedate, and the other from here)
        # TODO when #112 is implemented,
        # this will result in only the whole dataset being versioned

        col = self.files[src_key]['columns']
        with open(raw, 'r', encoding="utf8") as tsvfile:
            reader = csv.reader(tsvfile, delimiter='\t', quotechar='\"')
            row = next(reader)  # drop Description
            row = str(next(reader))[9:19]
            LOG.info("Ingest from %s", row)
            date = datetime.strptime(
                row.strip(), '%Y-%m-%d').strftime("%Y-%m-%d-%H-%M")

            if file_info.get("url") is not None:
                self.dataset.set_ingest_source_file_version_date(
                    file_info.get("url"), date)

            row = next(reader)  # drop tracker url
{'fraction_non_alphanumeric': 0.10547178610290689, 'fraction_numerical': 0.01010415047411783, 'mean_word_length': 2.070150322118826, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 7, 'https://': 10, 'lorem ipsum': 0, 'www.': 6, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1663750', 'n_tokens_mistral': 4008, 'n_tokens_neox': 3707, 'n_words': 972}