<commit_before>#!/usr/bin/env python
# -*- coding: utf-8 -*-

"""
General utility functions.
"""

__all__ = ["label_vector", "progressbar", "short_hash"]

import logging
import numpy as np
import sys
from time import time
from collections import (Counter, Iterable, OrderedDict)
from hashlib import md5
from itertools import combinations_with_replacement
from six import string_types

logger = logging.getLogger(__name__)


def short_hash(contents):
    """
    Return a short hash string of some iterable content.

    :param contents:
        The contents to calculate a hash for.

    :returns:
        A concatenated string of 10-character length hashes for all items in the
        contents provided.
    """
    if not isinstance(contents, Iterable): contents = [contents]
    return "".join([str(md5(str(item)).hexdigest())[:10] for item in contents])


def is_structured_label_vector(label_vector):
    """
    Return whether the provided label vector is structured correctly.
    """

    if not isinstance(label_vector, (list, tuple)):
        return False

    for descriptor in label_vector:
        if not isinstance(descriptor, (list, tuple)):
            return False

        for term in descriptor:
            if not isinstance(term, (list, tuple)) \
            or len(term) != 2 \
            or not isinstance(term[-1], (int, float)):
                return False

    if len(label_vector) == 0 or sum(map(len, label_vector)) == 0:
        return False

    return True


def parse_label_vector(label_vector_description, columns=None, **kwargs):
    """
    Return a structured form of a label vector from unstructured,
    human-readable input.
<commit_msg>Encode before hashing for python3<commit_after>#!/usr/bin/env python
# -*- coding: utf-8 -*-

"""
General utility functions.
"""

__all__ = ["label_vector", "progressbar", "short_hash"]

import logging
import numpy as np
import sys
from time import time
from collections import (Counter, Iterable, OrderedDict)
from hashlib import md5
from itertools import combinations_with_replacement
from six import string_types

logger = logging.getLogger(__name__)


def short_hash(contents):
    """
    Return a short hash string of some iterable content.

    :param contents:
        The contents to calculate a hash for.

    :returns:
        A concatenated string of 10-character length hashes for all items in the
        contents provided.
    """
    if not isinstance(contents, Iterable): contents = [contents]
    return "".join([str(md5(str(item).encode("utf-8")).hexdigest())[:10] \
        for item in contents])


def is_structured_label_vector(label_vector):
    """
    Return whether the provided label vector is structured correctly.
    """

    if not isinstance(label_vector, (list, tuple)):
        return False

    for descriptor in label_vector:
        if not isinstance(descriptor, (list, tuple)):
            return False

        for term in descriptor:
            if not isinstance(term, (list, tuple)) \
            or len(term) != 2 \
            or not isinstance(term[-1], (int, float)):
                return False

    if len(label_vector) == 0 or sum(map(len, label_vector)) == 0:
        return False

    return True


def parse_label_vector(label_vector_description, columns=None, **kwargs):
    """
    Return a structured form of a label vector from unstructured,
    human-readable input.

{'fraction_non_alphanumeric': 0.09606345475910694, 'fraction_numerical': 0.007050528789659225, 'mean_word_length': 3.068100358422939, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5175250', 'n_tokens_mistral': 953, 'n_tokens_neox': 908, 'n_words': 334}