<commit_before>#!/usr/bin/env python3.6
"""
    A collection of reused functions and classes.
"""

import os
import re
import asyncio
import inspect
from functools import wraps
from multiprocessing import Manager
import rdflib
from rdflib.extras import infixowl

rdflib.plugin.register('nifttl', rdflib.serializer.Serializer, 'pyontutils.ttlser', 'CustomTurtleSerializer')

def refile(thisFile, path):
    return os.path.join(os.path.dirname(thisFile), path)

def async_getter(function, listOfArgs):
    async def future_loop(future_):
        loop = asyncio.get_event_loop()
        futures = []
        for args in listOfArgs:
            future = loop.run_in_executor(None, function, *args)
            futures.append(future)
        print('Futures compiled')
        responses = []
        for f in futures:
            responses.append(await f)
        future_.set_result(responses)
    future = asyncio.Future()
    loop = asyncio.get_event_loop()
    loop.run_until_complete(future_loop(future))
    return future.result()

def mysql_conn_helper(host, db, user, port=3306):
    kwargs = {
        'host':host,
        'db':db,
        'user':user,
        'port':port,
        'password':None,  # no you may NOT pass it in
    }
    with open(os.path.expanduser('~/.mypass'), 'rt') as f:
        entries = [l.strip().split(':', 4) for l in f.readlines()]
    for e_host, e_port, e_db, e_user, e_pass in entries:
        e_port = int(e_port)
        if host == e_host:
            print('yes:', host)
            if  port == e_port:
                print('yes:', port)
                if db == e_db or e_db == '*':  # FIXME bad * expansion
                    print('yes:', db)
                    if user == e_user:
                        print('yes:', user)
                        kwargs['password'] = e_pass  # last entry wins
    e_pass = None
    if kwargs['password'] is None:
        raise ConnectionError('No password as found for {user}@{host}:{port}/{db}'.format(**kwargs))

    return kwargs

def _loadPrefixes():
    import yaml
    try:
        with open(os.path.expanduser('~/git/NIF-Ontology/scigraph/nifstd_curie_map.yaml'), 'rt') as f:
            curie_map = yaml.load(f)
    except FileNotFoundError:
        import requests
        curie_map = requests.get('https://github.com/SciCrunch/NIF-Ontology/raw/master/scigraph/nifstd_curie_map.yaml')
        curie_map = yaml.load(curie_map.text)

    # holding place for values that are not in the curie map
    extras = {
        '':None,  # safety
        'ILXREPLACE':'http://ILXREPLACE.org/',
        'FIXME':'http://FIXME.org/',
        'ILX':'http://uri.interlex.org/base/ilx_', 
        'NIFTTL':'http://ontology.neuinfo.org/NIF/ttl/',
        'NIFSTD':'http://uri.neuinfo.org/nif/nifstd/',  # note that this is '' in real curies
        'NLXWIKI':'http://neurolex.org/wiki/',
        'hasRole':'http://purl.obolibrary.org/obo/RO_0000087',
        'dc':'http://purl.org/dc/elements/1.1/',
        'definition':'http://purl.obolibrary.org/obo/IAO_0000115',
        'ilx':'http://uri.interlex.org/base/', 
        'nsu':'http://www.FIXME.org/nsupper#',
        'oboInOwl':'http://www.geneontology.org/formats/oboInOwl#',
        'owl':'http://www.w3.org/2002/07/owl#',
        'replacedBy':'http://purl.obolibrary.org/obo/IAO_0100001',
        'ro':'http://www.obofoundry.org/ro/ro.owl#',
        'skos':'http://www.w3.org/2004/02/skos/core#',
        'rdfs':'http://www.w3.org/2000/01/rdf-schema#',
    }
    curie_map.update(extras)
    return curie_map

PREFIXES = _loadPrefixes()

def makePrefixes(*prefixes):
    return {k:PREFIXES[k] for k in prefixes}

class makeGraph:
    SYNONYM = 'OBOANN:synonym'  # dangerous with prefixes

    def __init__(self, name, prefixes=None, graph=None, writeloc='/tmp/'):
        self.name = name
        self.writeloc = writeloc
        self.namespaces = {}
        if prefixes:
            self.namespaces.update({p:rdflib.Namespace(ns) for p, ns in prefixes.items()})
        if graph:  # graph takes precidence
            self.namespaces.update({p:rdflib.Namespace(ns) for p, ns in graph.namespaces()})
        if not graph and not prefixes:
            raise ValueError('No prefixes or graph specified.')<commit_msg>utils added rdf and NIFRET to extras for makePrefixes<commit_after>#!/usr/bin/env python3.6
"""
    A collection of reused functions and classes.
"""

import os
import re
import asyncio
import inspect
from functools import wraps
from multiprocessing import Manager
import rdflib
from rdflib.extras import infixowl

rdflib.plugin.register('nifttl', rdflib.serializer.Serializer, 'pyontutils.ttlser', 'CustomTurtleSerializer')

def refile(thisFile, path):
    return os.path.join(os.path.dirname(thisFile), path)

def async_getter(function, listOfArgs):
    async def future_loop(future_):
        loop = asyncio.get_event_loop()
        futures = []
        for args in listOfArgs:
            future = loop.run_in_executor(None, function, *args)
            futures.append(future)
        print('Futures compiled')
        responses = []
        for f in futures:
            responses.append(await f)
        future_.set_result(responses)
    future = asyncio.Future()
    loop = asyncio.get_event_loop()
    loop.run_until_complete(future_loop(future))
    return future.result()

def mysql_conn_helper(host, db, user, port=3306):
    kwargs = {
        'host':host,
        'db':db,
        'user':user,
        'port':port,
        'password':<PASSWORD>,  # no you may NOT pass it in
    }
    with open(os.path.expanduser('~/.mypass'), 'rt') as f:
        entries = [l.strip().split(':', 4) for l in f.readlines()]
    for e_host, e_port, e_db, e_user, e_pass in entries:
        e_port = int(e_port)
        if host == e_host:
            print('yes:', host)
            if  port == e_port:
                print('yes:', port)
                if db == e_db or e_db == '*':  # FIXME bad * expansion
                    print('yes:', db)
                    if user == e_user:
                        print('yes:', user)
                        kwargs['password'] = e_pass  # last entry wins
    e_pass = None
    if kwargs['password'] is None:
        raise ConnectionError('No password as found for {user}@{host}:{port}/{db}'.format(**kwargs))

    return kwargs

def _loadPrefixes():
    import yaml
    try:
        with open(os.path.expanduser('~/git/NIF-Ontology/scigraph/nifstd_curie_map.yaml'), 'rt') as f:
            curie_map = yaml.load(f)
    except FileNotFoundError:
        import requests
        curie_map = requests.get('https://github.com/SciCrunch/NIF-Ontology/raw/master/scigraph/nifstd_curie_map.yaml')
        curie_map = yaml.load(curie_map.text)

    # holding place for values that are not in the curie map
    extras = {
        '':None,  # safety
        'ILXREPLACE':'http://ILXREPLACE.org/',
        'FIXME':'http://FIXME.org/',
        'ILX':'http://uri.interlex.org/base/ilx_', 
        'NIFTTL':'http://ontology.neuinfo.org/NIF/ttl/',
        'NIFSTD':'http://uri.neuinfo.org/nif/nifstd/',  # note that this is '' in real curies
        'NIFRET':'http://ontology.neuinfo.org/NIF/Retired/NIF-Retired.owl#',
        'NLXWIKI':'http://neurolex.org/wiki/',
        'hasRole':'http://purl.obolibrary.org/obo/RO_0000087',
        'dc':'http://purl.org/dc/elements/1.1/',
        'definition':'http://purl.obolibrary.org/obo/IAO_0000115',
        'ilx':'http://uri.interlex.org/base/', 
        'nsu':'http://www.FIXME.org/nsupper#',
        'oboInOwl':'http://www.geneontology.org/formats/oboInOwl#',
        'owl':'http://www.w3.org/2002/07/owl#',
        'replacedBy':'http://purl.obolibrary.org/obo/IAO_0100001',
        'ro':'http://www.obofoundry.org/ro/ro.owl#',
        'skos':'http://www.w3.org/2004/02/skos/core#',
        'rdf':'http://www.w3.org/1999/02/22-rdf-syntax-ns#',
        'rdfs':'http://www.w3.org/2000/01/rdf-schema#',
    }
    curie_map.update(extras)
    return curie_map

PREFIXES = _loadPrefixes()

def makePrefixes(*prefixes):
    return {k:PREFIXES[k] for k in prefixes}

class makeGraph:
    SYNONYM = 'OBOANN:synonym'  # dangerous with prefixes

    def __init__(self, name, prefixes=None, graph=None, writeloc='/tmp/'):
        self.name = name
        self.writeloc = writeloc
        self.namespaces = {}
        if prefixes:
            self.namespaces.update({p:rdflib.Namespace(ns) for p, ns in prefixes.items()})
        if graph:  # graph takes precidence
            self.namespaces.update({p:rdflib.Namespace(ns) for p, ns in graph.namespaces()})
        if not graph and not prefixes:
            raise ValueError('No prefixes or graph specified.')
{'fraction_non_alphanumeric': 0.147459584295612, 'fraction_numerical': 0.012817551963048499, 'mean_word_length': 2.5293398533007334, 'pattern_counts': {'":': 0, '<': 4, '<?xml version=': 0, '>': 4, 'https://': 2, 'lorem ipsum': 0, 'www.': 13, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '725112', 'n_tokens_mistral': 3116, 'n_tokens_neox': 2950, 'n_words': 571}