<commit_before>import logging

__all__ = ['seqToKV', 'kvToSeq', 'dictToKV', 'kvToDict']


class KVFormError(ValueError):
    pass


def seqToKV(seq, strict=False):
    """Represent a sequence of pairs of strings as newline-terminated
    key:value pairs. The pairs are generated in the order given.

    @param seq: The pairs
    @type seq: [(str, (unicode|str))]

    @return: A string representation of the sequence
    @rtype: str
    """
    def err(msg):
        formatted = 'seqToKV warning: %s: %r' % (msg, seq)
        if strict:
            raise KVFormError(formatted)
        else:
            logging.warn(formatted)

    lines = []
    for k, v in seq:
        if isinstance(k, bytes):
            k = k.decode('utf-8')
        elif not isinstance(k, str):
            err('Converting key to string: %r' % k)
            k = str(k)

        if '\n' in k:
            raise KVFormError(
                'Invalid input for seqToKV: key contains newline: %r' % (k,))

        if ':' in k:
            raise KVFormError(
                'Invalid input for seqToKV: key contains colon: %r' % (k,))

        if k.strip() != k:
            err('Key has whitespace at beginning or end: %r' % (k,))

        if isinstance(v, bytes):
            v = v.decode('UTF8')
        elif not isinstance(v, str):
            err('Converting value to string: %r' % (v,))
            v = str(v)

        if '\n' in v:
            raise KVFormError(
                'Invalid input for seqToKV: value contains newline: %r' % (v,))

        if v.strip() != v:
            err('Value has whitespace at beginning or end: %r' % (v,))

        lines.append(k + ':' + v + '\n')

    return ''.join(lines).encode('utf-8')


def kvToSeq(data, strict=False):
    """

    After one parse, seqToKV and kvToSeq are inverses, with no warnings::

        seq = kvToSeq(s)
        seqToKV(kvToSeq(seq)) == seq
    """
    def err(msg):
        formatted = 'kvToSeq warning: %s: %r' % (msg, data)
        if strict:
            raise KVFormError(formatted)
        else:
            logging.warn(formatted)<commit_msg>Add comments to confirm return types<commit_after>import logging

__all__ = ['seqToKV', 'kvToSeq', 'dictToKV', 'kvToDict']


class KVFormError(ValueError):
    pass


def seqToKV(seq, strict=False):
    """Represent a sequence of pairs of strings as newline-terminated
    key:value pairs. The pairs are generated in the order given.

    @param seq: The pairs
    @type seq: [(str, (unicode|str))]

    @return: A string representation of the sequence
    @rtype: bytes
    """
    def err(msg):
        formatted = 'seqToKV warning: %s: %r' % (msg, seq)
        if strict:
            raise KVFormError(formatted)
        else:
            logging.warn(formatted)

    lines = []
    for k, v in seq:
        if isinstance(k, bytes):
            k = k.decode('utf-8')
        elif not isinstance(k, str):
            err('Converting key to string: %r' % k)
            k = str(k)

        if '\n' in k:
            raise KVFormError(
                'Invalid input for seqToKV: key contains newline: %r' % (k,))

        if ':' in k:
            raise KVFormError(
                'Invalid input for seqToKV: key contains colon: %r' % (k,))

        if k.strip() != k:
            err('Key has whitespace at beginning or end: %r' % (k,))

        if isinstance(v, bytes):
            v = v.decode('utf-8')
        elif not isinstance(v, str):
            err('Converting value to string: %r' % (v,))
            v = str(v)

        if '\n' in v:
            raise KVFormError(
                'Invalid input for seqToKV: value contains newline: %r' % (v,))

        if v.strip() != v:
            err('Value has whitespace at beginning or end: %r' % (v,))

        lines.append(k + ':' + v + '\n')

    return ''.join(lines).encode('utf-8')


def kvToSeq(data, strict=False):
    """

    After one parse, seqToKV and kvToSeq are inverses, with no warnings::

        seq = kvToSeq(s)
        seqToKV(kvToSeq(seq)) == seq

    @return str
    """
    def err(msg):
        formatted = 'kvToSeq warning: %s: %r' % (msg, data)
        if strict:
            raise KVFormError(formatted)
        else:
            logging.warn(formatted)
{'fraction_non_alphanumeric': 0.1300543863797588, 'fraction_numerical': 0.0014187751241428233, 'mean_word_length': 1.9415855354659248, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2799060', 'n_tokens_mistral': 1395, 'n_tokens_neox': 1344, 'n_words': 426}