<commit_before>from itertools import combinations
import multiprocessing


from coalib.processes.SectionExecutor import get_cpu_count
from coalib.results.HiddenResult import HiddenResult
from coalib.settings.Setting import typed_dict
from coalib.bears.GlobalBear import GlobalBear
from bears.codeclone_detection.ClangCountVectorCreator import \
    ClangCountVectorCreator
from bears.codeclone_detection.ClangCountingConditions import condition_dict
from bears.codeclone_detection.CloneDetectionRoutines import \
    compare_functions, \
    get_count_matrices


"""
counting_condition_dict is a function object generated by typed_dict. This
function takes a setting and creates a dictionary out of it while it
converts all keys to counting condition function objects (via the
condition_dict) and all values to floats while unset values default to 1.
"""
counting_condition_dict = typed_dict(
    lambda setting: condition_dict[str(setting).lower()],
    float,
    1)


# Coverage cannot be measured because this is in another process
def get_difference(args):  # pragma: no cover
    """
    Retrieves the difference between two functions using the munkres algorithm.

    :param args: A tuple holding the first function id, the second and the
                 count matrices dictionary holding the count matrices for
                 each function with the function id as key.
    :return:     A tuple containing both function ids and their similarity.
    """
    function_1, function_2, count_matrices = args
    return (function_1,
            function_2,
            compare_functions(count_matrices[function_1],
                              count_matrices[function_2]))


class ClangSimilarityBear(GlobalBear):
    def run(self,
            condition_list: counting_condition_dict):
        '''
        Retrieves similarities for code clone detection. Those can be reused in
        another bear to produce results.

        :param condition_list:       A comma seperated list of counting
                                     conditions. Possible values are: used,
                                     returned, is_condition, in_condition,
                                     is_assignee, is_assigner, loop_content.
                                     Weightings can be assigned to each
                                     condition due to providing a dict
                                     value, i.e. having used weighted in
                                     half as much as other conditions would
                                     simply be: "used: 0.5, is_assignee".
                                     Weightings default to 1 if unset.
        '''
        if not isinstance(condition_list, dict):
            self.err("The condition_list setting is invalid. Code clone "
                     "detection cannot run.")
            return

        self.debug("Using the following counting conditions:")
        for key, val in condition_list.items():
            self.debug(" *", key.__name__, "(weighting: {})".format(val))

        self.debug("Creating count matrices...")
        count_matrices = get_count_matrices(
            ClangCountVectorCreator(list(condition_list.keys()),
                                    list(condition_list.values())),
            list(self.file_dict.keys()),
            lambda prog: self.debug("{:2.4f}%...".format(prog)))

        self.debug("Calculating differences...")
        # Code clone detection may take ages for a larger code basis. It is
        # highly probable, that no other bears are running in parallel,
        # thus we do parallel execution within this bear.
        pool = multiprocessing.Pool(get_cpu_count())

        differences = []
        function_count = len(count_matrices)
        # Thats n over 2, hardcoded to simplify calculation
        combination_length = function_count * (function_count-1) / 2
        function_combinations = [(f1, f2, count_matrices)
                                 for f1, f2 in combinations(count_matrices, 2)]

        for i, elem in enumerate(pool.imap_unordered(get_difference,
                                                     function_combinations,
                                                     chunksize=100)):
            if i % 1000 == 0:
                self.debug("{:2.4f}%...".format(100*i/combination_length))
            differences.append(elem)

        return [HiddenResult(self.__class__.__name__, differences)]
<commit_msg>ClangSimilarityBear: Documentation correction<commit_after>from itertools import combinations
import multiprocessing


from coalib.processes.SectionExecutor import get_cpu_count
from coalib.results.HiddenResult import HiddenResult
from coalib.settings.Setting import typed_dict
from coalib.bears.GlobalBear import GlobalBear
from bears.codeclone_detection.ClangCountVectorCreator import \
    ClangCountVectorCreator
from bears.codeclone_detection.ClangCountingConditions import condition_dict
from bears.codeclone_detection.CloneDetectionRoutines import \
    compare_functions, \
    get_count_matrices


"""
counting_condition_dict is a function object generated by typed_dict. This
function takes a setting and creates a dictionary out of it while it
converts all keys to counting condition function objects (via the
condition_dict) and all values to floats while unset values default to 1.
"""
counting_condition_dict = typed_dict(
    lambda setting: condition_dict[str(setting).lower()],
    float,
    1)


# Coverage cannot be measured because this is in another process
def get_difference(args):  # pragma: no cover
    """
    Retrieves the difference between two functions using the munkres algorithm.

    :param args: A tuple holding the first function id, the second and the
                 count matrices dictionary holding the count matrices for
                 each function with the function id as key.
    :return:     A tuple containing both function ids and their difference.
    """
    function_1, function_2, count_matrices = args
    return (function_1,
            function_2,
            compare_functions(count_matrices[function_1],
                              count_matrices[function_2]))


class ClangSimilarityBear(GlobalBear):
    def run(self,
            condition_list: counting_condition_dict):
        '''
        Retrieves similarities for code clone detection. Those can be reused in
        another bear to produce results.

        :param condition_list:       A comma seperated list of counting
                                     conditions. Possible values are: used,
                                     returned, is_condition, in_condition,
                                     is_assignee, is_assigner, loop_content.
                                     Weightings can be assigned to each
                                     condition due to providing a dict
                                     value, i.e. having used weighted in
                                     half as much as other conditions would
                                     simply be: "used: 0.5, is_assignee".
                                     Weightings default to 1 if unset.
        '''
        if not isinstance(condition_list, dict):
            self.err("The condition_list setting is invalid. Code clone "
                     "detection cannot run.")
            return

        self.debug("Using the following counting conditions:")
        for key, val in condition_list.items():
            self.debug(" *", key.__name__, "(weighting: {})".format(val))

        self.debug("Creating count matrices...")
        count_matrices = get_count_matrices(
            ClangCountVectorCreator(list(condition_list.keys()),
                                    list(condition_list.values())),
            list(self.file_dict.keys()),
            lambda prog: self.debug("{:2.4f}%...".format(prog)))

        self.debug("Calculating differences...")
        # Code clone detection may take ages for a larger code basis. It is
        # highly probable, that no other bears are running in parallel,
        # thus we do parallel execution within this bear.
        pool = multiprocessing.Pool(get_cpu_count())

        differences = []
        function_count = len(count_matrices)
        # Thats n over 2, hardcoded to simplify calculation
        combination_length = function_count * (function_count-1) / 2
        function_combinations = [(f1, f2, count_matrices)
                                 for f1, f2 in combinations(count_matrices, 2)]

        for i, elem in enumerate(pool.imap_unordered(get_difference,
                                                     function_combinations,
                                                     chunksize=100)):
            if i % 1000 == 0:
                self.debug("{:2.4f}%...".format(100*i/combination_length))
            differences.append(elem)

        return [HiddenResult(self.__class__.__name__, differences)]

{'fraction_non_alphanumeric': 0.06610925306577481, 'fraction_numerical': 0.00758082497212932, 'mean_word_length': 2.0775300171526587, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 32, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4816270', 'n_tokens_mistral': 2224, 'n_tokens_neox': 2105, 'n_words': 710}