<commit_before># -*- coding: utf-8 -*-
# vim: sw=4:ts=4:expandtab
"""
riko.bado.util
~~~~~~~~~~~~~~
Provides functions for creating asynchronous riko pipes

Examples:
    basic usage::

        >>> from riko import get_path
        >>> from riko.bado.util import xml2etree
"""
from __future__ import (
    absolute_import, division, print_function, unicode_literals)

from os import environ
from sys import executable
from functools import partial
from html.entities import entitydefs, name2codepoint

from builtins import *

from riko.lib.utils import _make_content

try:
    from twisted.internet.defer import maybeDeferred, Deferred
except ImportError:
    maybeDeferred = lambda *args: None
else:
    from twisted.internet import defer
    from twisted.internet.utils import getProcessOutput
    from twisted.internet.reactor import callLater

    from . import microdom
    from .microdom import EntityReference

    async_none = defer.succeed(None)
    async_return = partial(defer.succeed)
    async_partial = lambda f, **kwargs: partial(maybeDeferred, f, **kwargs)

DEF2NAME = {v: k for k, v in entitydefs.items()}


def async_sleep(seconds):
    d = Deferred()
    callLater(seconds, d.callback, None)
    return d


def defer_to_process(command):
    return getProcessOutput(executable, ['-c', command], environ)


def def2unicode(entitydef):
    """Convert an HTML entity reference into unicode.
    Double check if I need this since it seems to convert the input back into
    itself!
    """
    try:
        name = DEF2NAME[entitydef]
    except KeyError:
        cp = int(entitydef.lstrip('&#').rstrip(';'))
    else:
        cp = name2codepoint[name]

    return chr(cp)


def xml2etree(f, xml=True):
    readable = hasattr(f, 'read')

    if xml and readable:
        parse = microdom.parseXML
    elif readable:
        parse = partial(microdom.parse, lenient=True)
    elif xml:
        parse = microdom.parseXMLString
    else:
        parse = partial(microdom.parseString, lenient=True)

    return parse(f)
<commit_msg>Remove unnecessary entity conversion<commit_after># -*- coding: utf-8 -*-
# vim: sw=4:ts=4:expandtab
"""
riko.bado.util
~~~~~~~~~~~~~~
Provides functions for creating asynchronous riko pipes

Examples:
    basic usage::

        >>> from riko import get_path
        >>> from riko.bado.util import xml2etree
"""
from __future__ import (
    absolute_import, division, print_function, unicode_literals)

from os import environ
from sys import executable
from functools import partial
from html.entities import name2codepoint

from builtins import *

from riko.lib.utils import _make_content

try:
    from twisted.internet.defer import maybeDeferred, Deferred
except ImportError:
    maybeDeferred = lambda *args: None
else:
    from twisted.internet import defer
    from twisted.internet.utils import getProcessOutput
    from twisted.internet.reactor import callLater

    from . import microdom
    from .microdom import EntityReference

    async_none = defer.succeed(None)
    async_return = partial(defer.succeed)
    async_partial = lambda f, **kwargs: partial(maybeDeferred, f, **kwargs)



def async_sleep(seconds):
    d = Deferred()
    callLater(seconds, d.callback, None)
    return d


def defer_to_process(command):
    return getProcessOutput(executable, ['-c', command], environ)


def def2unicode(entitydef):
    """Convert an HTML entity reference into unicode.
    """
    cleaned = entitydef.lstrip('&').lstrip('#').rstrip(';')
    cp = int(cleaned) if '&#' in entitydef else name2codepoint[cleaned]
    return chr(cp)


def xml2etree(f, xml=True):
    readable = hasattr(f, 'read')

    if xml and readable:
        parse = microdom.parseXML
    elif readable:
        parse = partial(microdom.parse, lenient=True)
    elif xml:
        parse = microdom.parseXMLString
    else:
        parse = partial(microdom.parseString, lenient=True)

    return parse(f)

{'fraction_non_alphanumeric': 0.09966862095335202, 'fraction_numerical': 0.004588325261279633, 'mean_word_length': 3.5948477751756442, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 15, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 10}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7441028', 'n_tokens_mistral': 1221, 'n_tokens_neox': 1171, 'n_words': 349}