<commit_before>import logging
import requests
from dipper.sources.Source import Source
from dipper.models.Dataset import Dataset
from dipper.models.assoc.Association import Assoc
from dipper.models.Evidence import Evidence
from dipper.models.Provenance import Provenance
from dipper import curie_map
from dipper.utils.GraphUtils import GraphUtils
from pathlib import Path
import json

logger = logging.getLogger(__name__)


class MyDrug(Source):
    """
    Drugs and Compounds stored in the BioThings database
    """
    MY_DRUG_API = 'http://c.biothings.io/v1/query'

    files = {
        'aeolus': {
            'file': 'aeolus.json'
        }
    }

    def __init__(self):
        super().__init__('mydrug')
        self.load_bindings()

        self.dataset = Dataset(
            'MyDrug', 'Drugs and Compounds in BioThings',
            'http://c.biothings.io/')

    def fetch(self, is_dl_forced=False):
        """
        Note there is a unpublished mydrug client that works like this:
        from mydrug import MyDrugInfo
        md = MyDrugInfo()
        r = list(md.query('_exists_:aeolus', fetch_all=True))

        :param is_dl_forced: boolean, force download
        :return:
        """
        dir_path = Path(self.rawdir)
        aeolus_file = dir_path / self.files['aeolus']['file']
        if self.checkIfRemoteIsNewer(aeolus_file):
            aeolis_fh = aeolus_file.open('w')
            aeolis_fh.write("[")
            params = {
                'q': '_exists_:aeolus',
                'from': 0,
                'rows': 10,
                'fetch_all': 'TRUE'
            }
            result_count = params['rows']

            while params['from'] < result_count:
                solr_request = requests.get(self.MY_DRUG_API, params=params)
                response = solr_request.json()
                for index, doc in enumerate(response['hits']):
                    if params['from'] == 0 and index == 0:
                        aeolis_fh.write("{}".format(json.dumps(doc)))
                    else:
                        aeolis_fh.write(",\n{}".format(json.dumps(doc)))
                if params['from'] % 500 == 0:
                    logger.info("Fetched {} documents".format(params['from']))
                result_count = response['total']
                params['from'] += params['rows']

            aeolis_fh.write("\n]")
            aeolis_fh.close()

    def parse(self, limit=None, or_limit=1):
        """
        Parse mydrug files
        :param limit: int limit json docs processed
        :param or_limit: int odds ratio limit
        :return: None
        """
        dir_path = Path(self.rawdir)
        aeolus_file = dir_path / self.files['aeolus']['file']
        aeolus_fh = aeolus_file.open('r')
        count = 0
        for line in aeolus_fh.readlines():
            if limit is not None and count >= limit:
                break
            line = line.rstrip("\n,")
            if line != '[' and line != ']':
                self._parse_aeolus_data(document=json.loads(line),
                                        or_limit=or_limit)
                count += 1
            if count % 500 == 0:
                    logger.info("Processed {} documents".format(count))

        aeolus_fh.close()
        return

    def _parse_aeolus_data(self, document, or_limit=None):
        graph_util = GraphUtils(curie_map.get())

        rxcui_curie = "RXCUI:{}".format(document['aeolus']['rxcui'])
        uni_curie = "UNII:{}".format(document['aeolus']['unii'])
        graph_util.addEntityToGraph(self.graph, rxcui_curie,
                                    document['aeolus']['drug_name'])
        graph_util.addEntityToGraph(self.graph, uni_curie,
                                    document['aeolus']['drug_name'])

        graph_util.addSameIndividual(self.graph, rxcui_curie, uni_curie)
        graph_util.addTriple(self.graph, rxcui_curie,
                             graph_util.annotation_properties['inchi_key'],
                             document['unii']['inchikey'],
                             object_is_literal=True)

        if or_limit is not None:
            outcomes = (outcome for outcome in document['aeolus']['outcomes']<commit_msg>remove fetch_all param<commit_after>import logging
import requests
from dipper.sources.Source import Source
from dipper.models.Dataset import Dataset
from dipper.models.assoc.Association import Assoc
from dipper.models.Evidence import Evidence
from dipper.models.Provenance import Provenance
from dipper import curie_map
from dipper.utils.GraphUtils import GraphUtils
from pathlib import Path
import json

logger = logging.getLogger(__name__)


class MyDrug(Source):
    """
    Drugs and Compounds stored in the BioThings database
    """
    MY_DRUG_API = 'http://c.biothings.io/v1/query'

    files = {
        'aeolus': {
            'file': 'aeolus.json'
        }
    }

    def __init__(self):
        super().__init__('mydrug')
        self.load_bindings()

        self.dataset = Dataset(
            'MyDrug', 'Drugs and Compounds in BioThings',
            'http://c.biothings.io/')

    def fetch(self, is_dl_forced=False):
        """
        Note there is a unpublished mydrug client that works like this:
        from mydrug import MyDrugInfo
        md = MyDrugInfo()
        r = list(md.query('_exists_:aeolus', fetch_all=True))

        :param is_dl_forced: boolean, force download
        :return:
        """
        dir_path = Path(self.rawdir)
        aeolus_file = dir_path / self.files['aeolus']['file']
        if self.checkIfRemoteIsNewer(aeolus_file):
            aeolis_fh = aeolus_file.open('w')
            aeolis_fh.write("[")
            params = {
                'q': '_exists_:aeolus',
                'from': 0,
                'rows': 10
            }
            result_count = params['rows']

            while params['from'] < result_count:
                solr_request = requests.get(self.MY_DRUG_API, params=params)
                response = solr_request.json()
                for index, doc in enumerate(response['hits']):
                    if params['from'] == 0 and index == 0:
                        aeolis_fh.write("{}".format(json.dumps(doc)))
                    else:
                        aeolis_fh.write(",\n{}".format(json.dumps(doc)))
                if params['from'] % 500 == 0:
                    logger.info("Fetched {} documents".format(params['from']))
                result_count = response['total']
                params['from'] += params['rows']

            aeolis_fh.write("\n]")
            aeolis_fh.close()

    def parse(self, limit=None, or_limit=1):
        """
        Parse mydrug files
        :param limit: int limit json docs processed
        :param or_limit: int odds ratio limit
        :return: None
        """
        dir_path = Path(self.rawdir)
        aeolus_file = dir_path / self.files['aeolus']['file']
        aeolus_fh = aeolus_file.open('r')
        count = 0
        for line in aeolus_fh.readlines():
            if limit is not None and count >= limit:
                break
            line = line.rstrip("\n,")
            if line != '[' and line != ']':
                self._parse_aeolus_data(document=json.loads(line),
                                        or_limit=or_limit)
                count += 1
            if count % 500 == 0:
                    logger.info("Processed {} documents".format(count))

        aeolus_fh.close()
        return

    def _parse_aeolus_data(self, document, or_limit=None):
        print(document['aeolus']['rxcui'])
        graph_util = GraphUtils(curie_map.get())

        rxcui_curie = "RXCUI:{}".format(document['aeolus']['rxcui'])
        uni_curie = "UNII:{}".format(document['aeolus']['unii'])
        graph_util.addEntityToGraph(self.graph, rxcui_curie,
                                    document['aeolus']['drug_name'])
        graph_util.addEntityToGraph(self.graph, uni_curie,
                                    document['aeolus']['drug_name'])

        graph_util.addSameIndividual(self.graph, rxcui_curie, uni_curie)
        graph_util.addTriple(self.graph, rxcui_curie,
                             graph_util.annotation_properties['inchi_key'],
                             document['unii']['inchikey'],
                             object_is_literal=True)

        if or_limit is not None:
            outcomes = (outcome for outcome in document['aeolus']['outcomes']
{'fraction_non_alphanumeric': 0.11690924150988048, 'fraction_numerical': 0.004023192521595077, 'mean_word_length': 1.9084652443220922, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 8, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6578343', 'n_tokens_mistral': 2615, 'n_tokens_neox': 2431, 'n_words': 500}