<commit_before>#!/usr/bin/env python3.5

import os
from glob import glob
from rdflib.namespace import SKOS
from parcellation import OntMeta, TODAY
from utils import makeGraph, makePrefixes

PREFIXES = makePrefixes('SCR', 'NIFMOL', 'NIFNEURON', 'NIFCELL', 'NIFGA', 'UBERON', 'PR', 'NIFNEURMOR', 'skos', 'owl')

ont = OntMeta('http://ontology.neuinfo.org/NIF/ttl/',
              'ksdesc-defs',
              'Knolwedge Space Defs',
              'KSDEFS',
              'Definitions from knowledge space descriptions. Generated by pyontutils/ksdesc_bridge.py',
              TODAY)

ontid = ont.path + ont.filename + '.ttl'
g = makeGraph(ont.filename, prefixes=PREFIXES)
g.add_ont(ontid, *ont[2:])

top_level = glob(os.path.expanduser('~/git/ksdesc/') + '*')

id_def_dict = {}
for putative_dir in top_level:
    if os.path.isdir(putative_dir):
        for putative_md in glob(putative_dir + '/*.md'):
            ident = os.path.split(putative_dir)[-1] + ':' + os.path.splitext(os.path.split(putative_md)[-1])[0]
            print(ident)
            with open(putative_md, 'rt') as f:
                def_ = f.read()

            if 'Definition' in def_:
                def_ = def_.split('Definition', 1)[-1].strip().strip('=').strip()

            g.add_node(ident, SKOS.definition, def_)

g.write()
<commit_msg>ksdesc_bridge corrected ontology location and added metadata<commit_after>#!/usr/bin/env python3.5

import os
from glob import glob
from rdflib.namespace import SKOS
from parcellation import OntMeta, TODAY
from utils import makeGraph, makePrefixes

PREFIXES = makePrefixes('SCR', 'NIFMOL', 'NIFNEURON', 'NIFCELL', 'NIFGA', 'UBERON', 'PR', 'NIFNEURMOR', 'skos', 'owl')

ont = OntMeta('http://ontology.neuinfo.org/NIF/ttl/generated/',
              'ksdesc-defs',
              'Knolwedge Space Defs',
              'KSDEFS',
              'Definitions from knowledge space descriptions. Generated by pyontutils/ksdesc_bridge.py',
              TODAY)

ontid = ont.path + ont.filename + '.ttl'
g = makeGraph(ont.filename, prefixes=PREFIXES)
g.add_ont(ontid, *ont[2:])

top_level = glob(os.path.expanduser('~/git/ksdesc/') + '*')

for putative_dir in top_level:
    if os.path.isdir(putative_dir):
        for putative_md in glob(putative_dir + '/*.md'):
            ident = os.path.split(putative_dir)[-1] + ':' + os.path.splitext(os.path.split(putative_md)[-1])[0]
            print(ident)
            with open(putative_md, 'rt') as f:
                def_ = f.read()

            if 'Definition' in def_:
                def_ = def_.split('Definition', 1)[-1].strip().strip('=').strip()

            g.add_node(ident, SKOS.definition, def_)

g.write()

{'fraction_non_alphanumeric': 0.15153788447111777, 'fraction_numerical': 0.006001500375093774, 'mean_word_length': 2.8708272859216257, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1151585', 'n_tokens_mistral': 957, 'n_tokens_neox': 919, 'n_words': 193}