<commit_before># encoding: utf-8

import os
import difflib
import argparse
import networkx
from lkbutils import (
    rdflib_load_terms,
    rdflib_load_relations,
    rdflib_to_networkx,
)
from lkbutils.nodeprovider import (
    merge_nodeproviders,
)
from lkbutils.relationprovider import (
    noconflict_providers,
)


def path_from_me(path):
    i_am_in = os.path.dirname(__file__)
    return os.path.sep.join([i_am_in, path])

BUILD_DESTINATION = path_from_me('./build/graph.dot')

TERMS_DIR = path_from_me('./source/terms')
RELATIONS_DIR = path_from_me('./source/relations')

TRACKING_LOG = path_from_me('./build/definitions.log')


def read_unicode(path, encoding='utf-8'):
    bytetext = open(path, 'rb').read()
    return bytetext.decode(encoding)

def yaml_files_in(directly):
    for root, dirs, files in os.walk(directly):
        for f in files:
            if f.endswith('.yml'):
                path = os.path.sep.join([root, f])
                print('    + .yml found: "{}"'.format(path))
                yield path

def yaml_texts_in(directly):
    for path in yaml_files_in(directly):
        yield read_unicode(path)

def get_node_provider(src_dir):
    node_providers = [
        rdflib_load_terms(yml).nodeprovider
        for yml in yaml_texts_in(src_dir)
    ]
    return merge_nodeproviders(*node_providers)

def get_relation_loaders(src_dir, nodeprovider=None):
    relation_loader_maps = [
        rdflib_load_relations(yml, nodeprovider=nodeprovider)
        for yml in yaml_texts_in(src_dir)
    ]

    sum_len_keys = sum([len(rlmap) for rlmap in relation_loader_maps])
    len_merged_keys = len(set(sum([list(rlmap.keys()) for rlmap in relation_loader_maps], [])))
    assert (sum_len_keys == len_merged_keys), 'multiple definitions for some relation'

    relation_loaders = sum(
        [
            list(rlmap[r] for r in sorted(rlmap))
            for rlmap in relation_loader_maps
        ],<commit_msg>Let link definitions separatable for same relation.<commit_after># encoding: utf-8

import os
import difflib
import argparse
import networkx
from lkbutils import (
    rdflib_load_terms,
    rdflib_load_relations,
    rdflib_to_networkx,
)
from lkbutils.nodeprovider import (
    merge_nodeproviders,
)
from lkbutils.relationprovider import (
    noconflict_providers,
)


def path_from_me(path):
    i_am_in = os.path.dirname(__file__)
    return os.path.sep.join([i_am_in, path])

BUILD_DESTINATION = path_from_me('./build/graph.dot')

TERMS_DIR = path_from_me('./source/terms')
RELATIONS_DIR = path_from_me('./source/relations')

TRACKING_LOG = path_from_me('./build/definitions.log')


def read_unicode(path, encoding='utf-8'):
    bytetext = open(path, 'rb').read()
    return bytetext.decode(encoding)

def yaml_files_in(directly):
    for root, dirs, files in os.walk(directly):
        for f in files:
            if f.endswith('.yml'):
                path = os.path.sep.join([root, f])
                print('    + .yml found: "{}"'.format(path))
                yield path

def yaml_texts_in(directly):
    for path in yaml_files_in(directly):
        yield read_unicode(path)

def get_node_provider(src_dir):
    node_providers = [
        rdflib_load_terms(yml).nodeprovider
        for yml in yaml_texts_in(src_dir)
    ]
    return merge_nodeproviders(*node_providers)

def get_relation_loaders(src_dir, nodeprovider=None):
    relation_loader_maps = [
        rdflib_load_relations(yml, nodeprovider=nodeprovider)
        for yml in yaml_texts_in(src_dir)
    ]

    relation_loaders = sum(
        [
            list(rlmap[r] for r in sorted(rlmap))
            for rlmap in relation_loader_maps
        ],
{'fraction_non_alphanumeric': 0.10483210483210484, 'fraction_numerical': 0.001092001092001092, 'mean_word_length': 3.351543942992874, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4397410', 'n_tokens_mistral': 1306, 'n_tokens_neox': 1266, 'n_words': 246}