<commit_before>def vectorized_evaluation(a, b, function, pre_function=None):
    """
    Uses the given function in the form of a lambda and produces a vectorized result if
    at least one of the items is an iterable value. Otherwise, it evaluates it as normal.
    :param a: The first value for the function
    :param b: The second value for the function
    :param function: A 2-arity lambda used for the function
    :param pre_function: A function that is called before vectorization (optional)
    :return: A value or array of values depending on a and b
    """

    if pre_function:
        if type(a) is list:
            a = [pre_function(x) for x in a]
        else:
            a = pre_function(a)

        if type(b) is list:
            b = [pre_function(x) for x in b]
        else:
            b = pre_function(b)

    # When both are lists
    if type(a) is list and type(b) is list:

        # Get the minimum and maximum of both lists in order to prevent index errors
        vector_range = min(len(a), len(b))
        max_range = max(len(a), len(b))
        vectorized_result = []

        # Compute the function for all in range elements
        for index in range(0, vector_range):
            vectorized_result.append(function(a[index], b[index]))

        # Append all out of range elements without being processed
        for index in range(vector_range, max_range):
            if len(a) == max_range:
                vectorized_result.append(a[index])
            else:
                vectorized_result.append(b[index])

        return vectorized_result<commit_msg>pep8 for lib/vectorizer.py<commit_after>def vectorized_evaluation(a, b, function, pre_function=None):
    """
    Uses the given function in the form of a lambda and produces a vectorized
    result if at least one of the items is an iterable value. Otherwise, it
    evaluates it as normal.
    :param a: The first value for the function
    :param b: The second value for the function
    :param function: A 2-arity lambda used for the function
    :param pre_function: A function that is called before vectorization
                         (optional)
    :return: A value or array of values depending on a and b
    """

    if pre_function:
        if type(a) is list:
            a = [pre_function(x) for x in a]
        else:
            a = pre_function(a)

        if type(b) is list:
            b = [pre_function(x) for x in b]
        else:
            b = pre_function(b)

    # When both are lists
    if type(a) is list and type(b) is list:

        # Get the minimum and maximum of both lists in order to prevent index
        # errors
        vector_range = min(len(a), len(b))
        max_range = max(len(a), len(b))
        vectorized_result = []

        # Compute the function for all in range elements
        for index in range(0, vector_range):
            vectorized_result.append(function(a[index], b[index]))

        # Append all out of range elements without being processed
        for index in range(vector_range, max_range):
            if len(a) == max_range:
                vectorized_result.append(a[index])
            else:
                vectorized_result.append(b[index])

        return vectorized_result
{'fraction_non_alphanumeric': 0.07438271604938272, 'fraction_numerical': 0.0015432098765432098, 'mean_word_length': 2.250752256770311, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1308344', 'n_tokens_mistral': 914, 'n_tokens_neox': 891, 'n_words': 365}