<commit_before>import os
from datetime import datetime
from stat import ST_CTIME
import re
import logging
import csv

from dipper.sources.PostgreSQLSource import PostgreSQLSource
from dipper.models.Model import Model
from dipper import config
from dipper.models.Reference import Reference


LOG = logging.getLogger(__name__)


class EOM(PostgreSQLSource):
    """
    Elements of Morphology is a resource from NHGRI that has definitions of
    morphological abnormalities, together with image depictions.
    We pull those relationships, as well as our local mapping of equivalences
    between EOM and HP terminologies.

    The website is crawled monthly by NIF's DISCO crawler system,
    which we utilize here.
    Be sure to have pg user/password connection details in your conf.yaml file,
    like:
    dbauth : {'disco' : {'user' : '<username>', 'password' : '<password>'}}

    Monarch-curated data for the HP to EOM mapping is stored at
    https://raw.githubusercontent.com/obophenotype/human-phenotype-ontology/master/src/mappings/hp-to-eom-mapping.tsv

    Since this resource is so small, the entirety of it is the "test" set.

    """

    # we are using the production view here; should we be using services?
    tables = [
        'dvp.pr_nlx_157874_1'
    ]

    GHRAW = 'https://raw.githubusercontent.com/obophenotype/human-phenotype-ontology'
    files = {
        'map': {  # in 2019 this repo has beem untouched in four years
            'file': 'hp-to-eom-mapping.tsv',
            'url': GHRAW + '/master/src/mappings/hp-to-eom-mapping.tsv',
            'columns': [
                'morphology_term_id',
                'morphology_term_label',
                'HP ID',
                'HP Label',
                'Notes'
            ]
        }
    }
    resources = {
        'tables': {
            'url': 'nif-db.crbs.ucsd.edu:5432',
            'file': 'dvp.pr_nlx_157874_1',
            'columns': [
                # head -1 dvp.pr_nlx_157874_1 | tr '\t' '\n' |sed "s|\(.*\)|'\1',|g"
                'morphology_term_id',
                'morphology_term_num',
                'morphology_term_label',
                'morphology_term_url',
                'terminology_category_label',
                'terminology_category_url',
                'subcategory',
                'objective_definition',
                'subjective_definition',
                'comments',
                'synonyms',
                'replaces',
                'small_figure_url',
                'large_figure_url',
                'e_uid',
                'v_uid',
                'v_uuid',
                'v_lastmodified',
                'v_status',
                'v_lastmodified_epoch'
            ],
        },
    }

    def __init__(self, graph_type, are_bnodes_skolemized):
        super().__init__(
            graph_type,
            are_bnodes_skolemized,
            'eom',
            ingest_title='Elements of Morphology',
            ingest_url='http://elementsofmorphology.nih.gov',
            data_rights='http://www.genome.gov/copyright.cfm',
            license_url='https://creativecommons.org/publicdomain/mark/1.0/'
            # file_handle=None
        )

    def fetch(self, is_dl_forced=False):
        '''connection details for DISCO'''
        cxn = {}
        cxn['host'] = 'nif-db.crbs.ucsd.edu'
        cxn['database'] = 'disco_crawler'
        cxn['port'] = '5432'
        cxn['user'] = config.get_config()['user']['disco']
        cxn['password'] = config.get_config()['keys'][cxn['user']]

        pg_iri = 'jdbc:postgresql://'+cxn['host']+':'+cxn['port']+'/'+cxn['database']
        self.dataset.set_ingest_source(pg_iri)

        # process the tables
        # self.fetch_from_pgdb(self.tables,cxn,100)  #for testing
        self.fetch_from_pgdb(self.tables, cxn)

        self.get_files(is_dl_forced)<commit_msg>Fixed postgres IRI<commit_after>import os
from datetime import datetime
from stat import ST_CTIME
import re
import logging
import csv

from dipper.sources.PostgreSQLSource import PostgreSQLSource
from dipper.models.Model import Model
from dipper import config
from dipper.models.Reference import Reference


LOG = logging.getLogger(__name__)


class EOM(PostgreSQLSource):
    """
    Elements of Morphology is a resource from NHGRI that has definitions of
    morphological abnormalities, together with image depictions.
    We pull those relationships, as well as our local mapping of equivalences
    between EOM and HP terminologies.

    The website is crawled monthly by NIF's DISCO crawler system,
    which we utilize here.
    Be sure to have pg user/password connection details in your conf.yaml file,
    like:
    dbauth : {'disco' : {'user' : '<username>', 'password' : '<password>'}}

    Monarch-curated data for the HP to EOM mapping is stored at
    https://raw.githubusercontent.com/obophenotype/human-phenotype-ontology/master/src/mappings/hp-to-eom-mapping.tsv

    Since this resource is so small, the entirety of it is the "test" set.

    """

    # we are using the production view here; should we be using services?
    tables = [
        'dvp.pr_nlx_157874_1'
    ]

    GHRAW = 'https://raw.githubusercontent.com/obophenotype/human-phenotype-ontology'
    files = {
        'map': {  # in 2019 this repo has beem untouched in four years
            'file': 'hp-to-eom-mapping.tsv',
            'url': GHRAW + '/master/src/mappings/hp-to-eom-mapping.tsv',
            'columns': [
                'morphology_term_id',
                'morphology_term_label',
                'HP ID',
                'HP Label',
                'Notes'
            ]
        }
    }
    resources = {
        'tables': {
            'url': 'nif-db.crbs.ucsd.edu:5432',
            'file': 'dvp.pr_nlx_157874_1',
            'columns': [
                # head -1 dvp.pr_nlx_157874_1 | tr '\t' '\n' |sed "s|\(.*\)|'\1',|g"
                'morphology_term_id',
                'morphology_term_num',
                'morphology_term_label',
                'morphology_term_url',
                'terminology_category_label',
                'terminology_category_url',
                'subcategory',
                'objective_definition',
                'subjective_definition',
                'comments',
                'synonyms',
                'replaces',
                'small_figure_url',
                'large_figure_url',
                'e_uid',
                'v_uid',
                'v_uuid',
                'v_lastmodified',
                'v_status',
                'v_lastmodified_epoch'
            ],
        },
    }

    def __init__(self, graph_type, are_bnodes_skolemized):
        super().__init__(
            graph_type,
            are_bnodes_skolemized,
            'eom',
            ingest_title='Elements of Morphology',
            ingest_url='http://elementsofmorphology.nih.gov',
            data_rights='http://www.genome.gov/copyright.cfm',
            license_url='https://creativecommons.org/publicdomain/mark/1.0/'
            # file_handle=None
        )

    def fetch(self, is_dl_forced=False):
        '''connection details for DISCO'''
        cxn = {}
        cxn['host'] = 'nif-db.crbs.ucsd.edu'
        cxn['database'] = 'disco_crawler'
        cxn['port'] = '5432'
        cxn['user'] = config.get_config()['user']['disco']
        cxn['password'] = config.get_config()['keys'][cxn['user']]

        pg_iri = 'postgresql://'+cxn['host']+':'+cxn['port']+'/'+cxn['database']
        self.dataset.set_ingest_source(pg_iri)

        # process the tables
        # self.fetch_from_pgdb(self.tables,cxn,100)  #for testing
        self.fetch_from_pgdb(self.tables, cxn)

        self.get_files(is_dl_forced)
{'fraction_non_alphanumeric': 0.11546418412205844, 'fraction_numerical': 0.01034393586759762, 'mean_word_length': 2.0345233424872498, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 7, 'https://': 6, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1738875', 'n_tokens_mistral': 2482, 'n_tokens_neox': 2307, 'n_words': 503}