<commit_before># encoding: utf-8

import yaml
from lkbutils import RDFLibNodeProvider, RDFLibRelationProvider


def parse_yaml(yaml_data):
    """Parse YAML stiring by PyYAML."""
    data = yaml.load(yaml_data)
    if data is None:
        return {}
    return data

def leaves_from_struct(data):
    """
    Traverse/generate structured data.

        data: list or dict is acceptable / any dict
              structure will be ignored, thus every list
              in deepest levels is enumerated as term list.
    """
    if isinstance(data, unicode):
        yield data
    elif isinstance(data, str):
        yield unicode(data)
    elif isinstance(data, list):
        for child in data:
            for item in leaves_from_struct(child):
                yield item
    elif isinstance(data, dict):
        for child in data.values():
            for item in leaves_from_struct(child):
                yield item
    else:
        raise TypeError(
            u'invalid data type: {}({})'.format(
                type(data), data
            )
        )


class TermLoader(object):
    """Loads term definitions."""

    def __init__(self, **options):
        """Loads term definitions."""
        self._nodeprovider = self.get_node_provider(**options)

    @property
    def ns(self):
        """Proxy to TermLoader._nodeprovider.ns."""
        return self._nodeprovider.ns

    @property
    def graph(self):
        """Proxy to TermLoader._nodeprovider.graph."""
        return self._nodeprovider.graph

    def get_node_provider(self, **options):
        """Delegate NodeProvider creation to subclasses."""
        return self.nodeprovider_class(**options)

    def load(self, data, as_property=False):
        """
        Load terms from data.

        data: list or dict is acceptable / any dict
              structure will be ignored, thus every list<commit_msg>Shortcut property for TermLoader._nodeprovider.<commit_after># encoding: utf-8

import yaml
from lkbutils import RDFLibNodeProvider, RDFLibRelationProvider


def parse_yaml(yaml_data):
    """Parse YAML stiring by PyYAML."""
    data = yaml.load(yaml_data)
    if data is None:
        return {}
    return data

def leaves_from_struct(data):
    """
    Traverse/generate structured data.

        data: list or dict is acceptable / any dict
              structure will be ignored, thus every list
              in deepest levels is enumerated as term list.
    """
    if isinstance(data, unicode):
        yield data
    elif isinstance(data, str):
        yield unicode(data)
    elif isinstance(data, list):
        for child in data:
            for item in leaves_from_struct(child):
                yield item
    elif isinstance(data, dict):
        for child in data.values():
            for item in leaves_from_struct(child):
                yield item
    else:
        raise TypeError(
            u'invalid data type: {}({})'.format(
                type(data), data
            )
        )


class TermLoader(object):
    """Loads term definitions."""

    def __init__(self, **options):
        """Loads term definitions."""
        self._nodeprovider = self.get_node_provider(**options)

    @property
    def ns(self):
        """Proxy to TermLoader._nodeprovider.ns."""
        return self._nodeprovider.ns

    @property
    def graph(self):
        """Proxy to TermLoader._nodeprovider.graph."""
        return self._nodeprovider.graph

    @property
    def nodeprovider(self):
        """Proxy to TermLoader._nodeprovider."""
        return self._nodeprovider

    def get_node_provider(self, **options):
        """Delegate NodeProvider creation to subclasses."""
        return self.nodeprovider_class(**options)

    def load(self, data, as_property=False):
        """
        Load terms from data.

        data: list or dict is acceptable / any dict
              structure will be ignored, thus every list
{'fraction_non_alphanumeric': 0.0942769545222279, 'fraction_numerical': 0.000510986203372509, 'mean_word_length': 2.2570715474209653, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1985955', 'n_tokens_mistral': 1052, 'n_tokens_neox': 1036, 'n_words': 332}