<commit_before># -*- coding: utf-8 -*-
"""These preprocessor functions are used to process data prior to output."""

from decimal import Decimal
import string

from cli_helpers import utils
from cli_helpers.compat import (StringIO, text_type, int_types, float_types,
                                HAS_PYGMENTS, Terminal256Formatter)


def convert_to_string(data, headers, **_):
    """Convert all *data* and *headers* to strings.

    Binary data that cannot be decoded is converted to a hexadecimal
    representation via :func:`binascii.hexlify`.

    :param iterable data: An :term:`iterable` (e.g. list) of rows.
    :param iterable headers: The column headers.
    :return: The processed data and headers.
    :rtype: tuple

    """
    return ([[utils.to_string(v) for v in row] for row in data],
            [utils.to_string(h) for h in headers])


def override_missing_value(data, headers, missing_value='', **_):
    """Override missing values in the *data* with *missing_value*.

    A missing value is any value that is :data:`None`.

    :param iterable data: An :term:`iterable` (e.g. list) of rows.
    :param iterable headers: The column headers.
    :param missing_value: The default value to use for missing data.
    :return: The processed data and headers.
    :rtype: tuple

    """
    return ([[missing_value if v is None else v for v in row] for row in data],
            headers)


def bytes_to_string(data, headers, **_):
    """Convert all *data* and *headers* bytes to strings.

    Binary data that cannot be decoded is converted to a hexadecimal
    representation via :func:`binascii.hexlify`.

    :param iterable data: An :term:`iterable` (e.g. list) of rows.
    :param iterable headers: The column headers.
    :return: The processed data and headers.
    :rtype: tuple

    """
    return ([[utils.bytes_to_string(v) for v in row] for row in data],
            [utils.bytes_to_string(h) for h in headers])


def align_decimals(data, headers, column_types=(), **_):
    """Align numbers in *data* on their decimal points.

    Whitespace padding is added before a number so that all numbers in a
    column are aligned.

    Outputting data before aligning the decimals::

        1
        2.1
        10.59

    Outputting data after aligning the decimals::

         1
         2.1
        10.59

    :param iterable data: An :term:`iterable` (e.g. list) of rows.
    :param iterable headers: The column headers.
    :param iterable column_types: The columns' type objects (e.g. int or float).
    :return: The processed data and headers.
    :rtype: tuple

    """
    pointpos = len(headers) * [0]
    for row in data:
        for i, v in enumerate(row):
            if column_types[i] is float and isinstance(v, Decimal):
                v = text_type(v)
                pointpos[i] = max(utils.intlen(v), pointpos[i])
    results = []
    for row in data:
        result = []
        for i, v in enumerate(row):
            if column_types[i] is float and isinstance(v, Decimal):
                v = text_type(v)
                result.append((pointpos[i] - utils.intlen(v)) * " " + v)
            else:
                result.append(v)
        results.append(result)
    return results, headers


def quote_whitespaces(data, headers, quotestyle="'", **_):
    """Quote leading/trailing whitespace in *data*.

    When outputing data with leading or trailing whitespace, it can be useful
    to put quotation marks around the value so the whitespace is more
    apparent. If one value in a column needs quoted, then all values in that
    column are quoted to keep things consistent.

    .. NOTE::<commit_msg>Align floats as well.<commit_after># -*- coding: utf-8 -*-
"""These preprocessor functions are used to process data prior to output."""

from decimal import Decimal
import string

from cli_helpers import utils
from cli_helpers.compat import (StringIO, text_type, int_types, float_types,
                                HAS_PYGMENTS, Terminal256Formatter)


def convert_to_string(data, headers, **_):
    """Convert all *data* and *headers* to strings.

    Binary data that cannot be decoded is converted to a hexadecimal
    representation via :func:`binascii.hexlify`.

    :param iterable data: An :term:`iterable` (e.g. list) of rows.
    :param iterable headers: The column headers.
    :return: The processed data and headers.
    :rtype: tuple

    """
    return ([[utils.to_string(v) for v in row] for row in data],
            [utils.to_string(h) for h in headers])


def override_missing_value(data, headers, missing_value='', **_):
    """Override missing values in the *data* with *missing_value*.

    A missing value is any value that is :data:`None`.

    :param iterable data: An :term:`iterable` (e.g. list) of rows.
    :param iterable headers: The column headers.
    :param missing_value: The default value to use for missing data.
    :return: The processed data and headers.
    :rtype: tuple

    """
    return ([[missing_value if v is None else v for v in row] for row in data],
            headers)


def bytes_to_string(data, headers, **_):
    """Convert all *data* and *headers* bytes to strings.

    Binary data that cannot be decoded is converted to a hexadecimal
    representation via :func:`binascii.hexlify`.

    :param iterable data: An :term:`iterable` (e.g. list) of rows.
    :param iterable headers: The column headers.
    :return: The processed data and headers.
    :rtype: tuple

    """
    return ([[utils.bytes_to_string(v) for v in row] for row in data],
            [utils.bytes_to_string(h) for h in headers])


def align_decimals(data, headers, column_types=(), **_):
    """Align numbers in *data* on their decimal points.

    Whitespace padding is added before a number so that all numbers in a
    column are aligned.

    Outputting data before aligning the decimals::

        1
        2.1
        10.59

    Outputting data after aligning the decimals::

         1
         2.1
        10.59

    :param iterable data: An :term:`iterable` (e.g. list) of rows.
    :param iterable headers: The column headers.
    :param iterable column_types: The columns' type objects (e.g. int or float).
    :return: The processed data and headers.
    :rtype: tuple

    """
    pointpos = len(headers) * [0]
    for row in data:
        for i, v in enumerate(row):
            if column_types[i] is float and type(v) in float_types:
                v = text_type(v)
                pointpos[i] = max(utils.intlen(v), pointpos[i])
    results = []
    for row in data:
        result = []
        for i, v in enumerate(row):
            if column_types[i] is float and type(v) in float_types:
                v = text_type(v)
                result.append((pointpos[i] - utils.intlen(v)) * " " + v)
            else:
                result.append(v)
        results.append(result)
    return results, headers


def quote_whitespaces(data, headers, quotestyle="'", **_):
    """Quote leading/trailing whitespace in *data*.

    When outputing data with leading or trailing whitespace, it can be useful
    to put quotation marks around the value so the whitespace is more
    apparent. If one value in a column needs quoted, then all values in that
    column are quoted to keep things consistent.

    .. NOTE::
{'fraction_non_alphanumeric': 0.09721082854799015, 'fraction_numerical': 0.005195515449822259, 'mean_word_length': 2.6629944917376065, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '122316', 'n_tokens_mistral': 2186, 'n_tokens_neox': 2093, 'n_words': 842}