<commit_before>"""
    libextract.formatters
    ~~~~~~~~~~~~~~~~~~~~~

    Formats the results of extraction into serializable
    representations, e.g. JSON, text.
"""

from functools import partial
try:
    from itertools import izip_longest as zip_longest
except ImportError:
    from itertools import zip_longest
from libextract.xpaths import FILTER_TEXT


UNLIMITED = float('NaN')


def get_text(node):
    """
    Gets the text contained within the children node
    of a given *node*, joined by a space.
    """
    return ' '.join(node.xpath(FILTER_TEXT))


def split_node_attr(node, attr):
    """
    Given a *node*, split the *attr* of the node
    into a list of strings, suitable for id/class
    handling.
    """
    return (node.get(attr) or '').split()


get_node_id = partial(split_node_attr, attr='id')
get_node_class = partial(split_node_attr, attr='class')


def node_json(node, depth=0):
    """
    Given a *node*, serialize it and recursively
    serialize it's children to a given *depth*.
    Note that if the *depth* runs out (goes to 0),
    the children key will be ``None``.
    """
    return {
        'xpath': node.getroottree().getpath(node),
        'class': get_node_class(node),
        'text': node.text,
        'tag': node.tag,
        'id': get_node_id(node),
        'children': (
            [node_json(n, depth-1) for n in node] if depth else None
        ),
    }


def chunks(iterable, size):
    """
    Yield successive chunks of *size* from a given
    *iterable*, filling the unfilled chunks with
    None.
    """
    args = [iter(iterable)] * size
    for row in zip_longest(*args, fillvalue=None):
        yield list(row)


def extract_tabular_node(node, tag):
    """
    Given a *node*, yield all of the child nodes in depth
    first order, that matches the given *tag*.
    """
    for elem in node.iter(tag):
        yield ' '.join(elem.text_content().split())


get_table_headings = partial(extract_tabular_node, tag='th')
get_table_rows = partial(extract_tabular_node, tag='td')


def table_json(node):
    """
    Given a table *node* (ie. <table>), return a dictionary
    of key-value pairs, where the key is the heading and the
    value is a list of rows.<commit_msg>switch to closures<commit_after>"""
    libextract.formatters
    ~~~~~~~~~~~~~~~~~~~~~

    Formats the results of extraction into serializable
    representations, e.g. JSON, text.
"""

try:
    from itertools import izip_longest as zip_longest
except ImportError:
    from itertools import zip_longest
from libextract.xpaths import FILTER_TEXT


UNLIMITED = float('NaN')


def get_text(node):
    """
    Gets the text contained within the children node
    of a given *node*, joined by a space.
    """
    return ' '.join(node.xpath(FILTER_TEXT))


def split_node_attr(attr):
    """
    Returns a function that, given a *node*, splits
    the *attr* of the node into a list of strings.
    """
    def splitter(node):
        return (node.get(attr) or '').split()
    return splitter


get_node_id = split_node_attr('id')
get_node_class = split_node_attr('class')


def node_json(node, depth=0):
    """
    Given a *node*, serialize it and recursively
    serialize it's children to a given *depth*.
    Note that if the *depth* runs out (goes to 0),
    the children key will be ``None``.
    """
    return {
        'xpath': node.getroottree().getpath(node),
        'class': get_node_class(node),
        'text': node.text,
        'tag': node.tag,
        'id': get_node_id(node),
        'children': (
            [node_json(n, depth-1) for n in node] if depth else None
        ),
    }


def chunks(iterable, size):
    """
    Yield successive chunks of *size* from a given
    *iterable*, filling the unfilled chunks with
    None.
    """
    args = [iter(iterable)] * size
    for row in zip_longest(*args, fillvalue=None):
        yield list(row)


def tabular_node_extractor(tag):
    """
    Returns a function that yields all of the child
    nodes of a given *node* in depth first order, that
    matches the given *tag*.
    """
    def extract(node):
        for elem in node.iter(tag):
            yield ' '.join(elem.text_content().split())
    return extract


get_table_headings = tabular_node_extractor('th')
get_table_rows = tabular_node_extractor('td')


def table_json(node):
    """
    Given a table *node* (ie. <table>), return a dictionary
    of key-value pairs, where the key is the heading and the
    value is a list of rows.
{'fraction_non_alphanumeric': 0.12044444444444445, 'fraction_numerical': 0.0013333333333333333, 'mean_word_length': 2.9586631486367634, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '738446', 'n_tokens_mistral': 1422, 'n_tokens_neox': 1396, 'n_words': 466}