<commit_before>import re
import hashlib
import os
import time
import logging
import urllib
import csv
from datetime import datetime
from stat import ST_CTIME, ST_SIZE
from inspect import getdoc
from rdflib import XSD, Literal

import yaml
from dipper.graph.RDFGraph import RDFGraph
from dipper.graph.StreamedGraph import StreamedGraph
from dipper.utils.GraphUtils import GraphUtils
from dipper.models.Dataset import Dataset

LOG = logging.getLogger(__name__)
CHUNK = 16 * 1024  # read remote urls of unknown size in 16k chunks
USER_AGENT = \
    "The Monarch Initiative (https://monarchinitiative.org/;<EMAIL>)"


class Source:
    """
    Abstract class for any data sources that we'll import and process.
    Each of the subclasses will fetch() the data, scrub() it as necessary,
    then parse() it into a graph.  The graph will then be written out to
    a single self.name().<dest_fmt>  file.

    Also provides a means to marshal metadata in a consistent fashion

    Houses the global translation table (from ontology label to ontology term)
    so it may as well be used everywhere.

    """
    DIPPERCACHE = 'https://archive.monarchinitiative.org/DipperCache'
    namespaces = {}
    files = {}
    ARGV = {}

    def __init__(
            self,
            graph_type='rdf_graph',     # or streamed_graph
            are_bnodes_skized=False,    # typically True
            data_release_version=None,
            name=None,                  # identifier; make an URI for nquads
            ingest_title=None,
            ingest_url=None,
            ingest_logo=None,     # this should be the name of file on 'MonarchLogoRepo'
            ingest_description=None,
            license_url=None,           # only if it is _our_ lic
            data_rights=None,           # their page that points to their current lic
            file_handle=None,
    ):

        # pull in the common test identifiers
        self.all_test_ids = self.open_and_parse_yaml('../../resources/test_ids.yaml')

        self.graph_type = graph_type
        self.are_bnodes_skized = are_bnodes_skized
        self.data_release_version = data_release_version
        self.ingest_title = ingest_title
        self.ingest_url = ingest_url
        self.ingest_logo = ingest_logo
        self.ingest_description = ingest_description
        self.license_url = license_url
        self.data_rights = data_rights
        self.localtt = self.load_local_translationtable(name)

        self.remote_file_timestamps = dict()

        if name is not None:
            self.name = name.lower()
        elif self.whoami() is not None:
            self.name = self.whoami().lower()

        LOG.info("Processing Source \"%s\"", self.name)
        self.test_only = False
        self.path = ""
        # to be used to store a subset of data for testing downstream.
        self.triple_count = 0
        self.outdir = 'out'
        self.testdir = 'tests'

        self.rawdir = '/'.join(('raw', self.name))
        self.testname = name + "_test"
        self.testfile = '/'.join((self.outdir, self.testname + ".ttl"))
        self.datasetfile = None

        # if raw data dir doesn't exist, create it
        if not os.path.exists(self.rawdir):
            os.makedirs(self.rawdir)
            raw_pth = os.path.abspath(self.rawdir)
            LOG.info("creating raw directory for %s at %s", self.name, raw_pth)
        # else:  # raw data dir does  exist. maybe should consider what is in it?

        # if output dir doesn't exist, create it
        if not os.path.exists(self.outdir):
            os.makedirs(self.outdir)
            out_pth = os.path.abspath(self.outdir)
            LOG.info("created output directory %s", out_pth)
        else:
            out_pth = os.path.abspath(self.outdir)

        LOG.info("Creating Test graph %s", self.testname)
        # note: tools such as protoge need skolemized blank nodes
        self.testgraph = RDFGraph(True, self.testname)

        if graph_type == 'rdf_graph':
            graph_id = ':MONARCH_' + str(self.name) + "_" + \
                datetime.now().isoformat(' ').split()[0]

            LOG.info("Creating graph  %s", graph_id)
            self.graph = RDFGraph(are_bnodes_skized, graph_id)

        elif graph_type == 'streamed_graph':
            # need to expand on export formats
            dest_file = open(out_pth + '/' + name + '.nt', 'w')   # where is the close?
            self.graph = StreamedGraph(are_bnodes_skized, dest_file)
            # leave test files as turtle (better human readibility)
        else:
            LOG.error(
                "%s graph type not supported\n"
                "valid types: rdf_graph, streamed_graph", graph_type)

        # pull in global ontology mapping datastructures
        self.globaltt = self.graph.globaltt
        self.globaltcid = self.graph.globaltcid

        self.curie_map = self.graph.curie_map
        # self.prefix_base = {v: k for k, v in self.curie_map.items()}

        # will be set to True if the intention is
        # to only process and write the test data
        self.test_only = False
        self.test_mode = False

        if self.ingest_description and getdoc(self) is not None:
            self.ingest_description = getdoc(self)

        self.dataset = Dataset(
            identifier=self.name,
            data_release_version=self.data_release_version,
            ingest_name=self.name,
            ingest_title=self.ingest_title,
            ingest_url=self.ingest_url,
            ingest_logo=self.ingest_logo,
            ingest_description=self.ingest_description,   # description
            license_url=self.license_url,    # only _OUR_ lic
            data_rights=self.data_rights,    # tries to point to others lics
            graph_type=graph_type,
            file_handle=file_handle
        )

    def fetch(self, is_dl_forced=False):
        """
        abstract method to fetch all data from an external resource.
        this should be overridden by subclasses
        :return: None

        """
        raise NotImplementedError

    def parse(self, limit):
        """
        abstract method to parse all data from an external resource,
        that was fetched in fetch() this should be overridden by subclasses<commit_msg>centralize common species<commit_after>import re
import hashlib
import os
import time
import logging
import urllib
import csv
from datetime import datetime
from stat import ST_CTIME, ST_SIZE
from inspect import getdoc
from rdflib import XSD, Literal

import yaml
from dipper.graph.RDFGraph import RDFGraph
from dipper.graph.StreamedGraph import StreamedGraph
from dipper.utils.GraphUtils import GraphUtils
from dipper.models.Dataset import Dataset

LOG = logging.getLogger(__name__)
CHUNK = 16 * 1024  # read remote urls of unknown size in 16k chunks
USER_AGENT = \
    "The Monarch Initiative (https://monarchinitiative.org/;<EMAIL>)"


class Source:
    """
    Abstract class for any data sources that we'll import and process.
    Each of the subclasses will fetch() the data, scrub() it as necessary,
    then parse() it into a graph.  The graph will then be written out to
    a single self.name().<dest_fmt>  file.

    Also provides a means to marshal metadata in a consistent fashion

    Houses the global translation table (from ontology label to ontology term)
    so it may as well be used everywhere.

    """
    DIPPERCACHE = 'https://archive.monarchinitiative.org/DipperCache'
    namespaces = {}
    files = {}
    ARGV = {}

    def __init__(
            self,
            graph_type='rdf_graph',     # or streamed_graph
            are_bnodes_skized=False,    # typically True
            data_release_version=None,
            name=None,                  # identifier; make an URI for nquads
            ingest_title=None,
            ingest_url=None,
            ingest_logo=None,     # this should be the name of file on 'MonarchLogoRepo'
            ingest_description=None,
            license_url=None,           # only if it is _our_ lic
            data_rights=None,           # their page that points to their current lic
            file_handle=None,
    ):

        # pull in the common test identifiers
        self.all_test_ids = self.open_and_parse_yaml('../../resources/test_ids.yaml')

        self.graph_type = graph_type
        self.are_bnodes_skized = are_bnodes_skized
        self.data_release_version = data_release_version
        self.ingest_title = ingest_title
        self.ingest_url = ingest_url
        self.ingest_logo = ingest_logo
        self.ingest_description = ingest_description
        self.license_url = license_url
        self.data_rights = data_rights
        self.localtt = self.load_local_translationtable(name)

        self.remote_file_timestamps = dict()

        if name is not None:
            self.name = name.lower()
        elif self.whoami() is not None:
            self.name = self.whoami().lower()

        LOG.info("Processing Source \"%s\"", self.name)
        self.test_only = False
        self.path = ""
        # to be used to store a subset of data for testing downstream.
        self.triple_count = 0
        self.outdir = 'out'
        self.testdir = 'tests'

        self.rawdir = '/'.join(('raw', self.name))
        self.testname = name + "_test"
        self.testfile = '/'.join((self.outdir, self.testname + ".ttl"))
        self.datasetfile = None

        # if raw data dir doesn't exist, create it
        if not os.path.exists(self.rawdir):
            os.makedirs(self.rawdir)
            raw_pth = os.path.abspath(self.rawdir)
            LOG.info("creating raw directory for %s at %s", self.name, raw_pth)
        # else:  # raw data dir does  exist. maybe should consider what is in it?

        # if output dir doesn't exist, create it
        if not os.path.exists(self.outdir):
            os.makedirs(self.outdir)
            out_pth = os.path.abspath(self.outdir)
            LOG.info("created output directory %s", out_pth)
        else:
            out_pth = os.path.abspath(self.outdir)

        LOG.info("Creating Test graph %s", self.testname)
        # note: tools such as protoge need skolemized blank nodes
        self.testgraph = RDFGraph(True, self.testname)

        if graph_type == 'rdf_graph':
            graph_id = ':MONARCH_' + str(self.name) + "_" + \
                datetime.now().isoformat(' ').split()[0]

            LOG.info("Creating graph  %s", graph_id)
            self.graph = RDFGraph(are_bnodes_skized, graph_id)

        elif graph_type == 'streamed_graph':
            # need to expand on export formats
            dest_file = open(out_pth + '/' + name + '.nt', 'w')   # where is the close?
            self.graph = StreamedGraph(are_bnodes_skized, dest_file)
            # leave test files as turtle (better human readibility)
        else:
            LOG.error(
                "%s graph type not supported\n"
                "valid types: rdf_graph, streamed_graph", graph_type)

        # pull in global ontology mapping datastructures
        self.globaltt = self.graph.globaltt
        self.globaltcid = self.graph.globaltcid

        self.curie_map = self.graph.curie_map
        # self.prefix_base = {v: k for k, v in self.curie_map.items()}

        # will be set to True if the intention is
        # to only process and write the test data
        self.test_only = False
        self.test_mode = False

        if self.ingest_description and getdoc(self) is not None:
            self.ingest_description = getdoc(self)

        self.dataset = Dataset(
            identifier=self.name,
            data_release_version=self.data_release_version,
            ingest_name=self.name,
            ingest_title=self.ingest_title,
            ingest_url=self.ingest_url,
            ingest_logo=self.ingest_logo,
            ingest_description=self.ingest_description,   # description
            license_url=self.license_url,    # only _OUR_ lic
            data_rights=self.data_rights,    # tries to point to others lics
            graph_type=graph_type,
            file_handle=file_handle
        )

        # see jenkins file   human, mouse, zebrafish, fly, worm        rat
        self.COMMON_TAXON = ['9606','10090','7955','7227','6239']  # '10116'

    def fetch(self, is_dl_forced=False):
        """
        abstract method to fetch all data from an external resource.
        this should be overridden by subclasses
        :return: None

        """
        raise NotImplementedError

    def parse(self, limit):
        """
        abstract method to parse all data from an external resource,
        that was fetched in fetch() this should be overridden by subclasses
{'fraction_non_alphanumeric': 0.08318235525797558, 'fraction_numerical': 0.0036234738085860574, 'mean_word_length': 2.2387755102040816, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 7, 'https://': 4, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1325208', 'n_tokens_mistral': 3786, 'n_tokens_neox': 3646, 'n_words': 1063}