<commit_before>from sklearn.metrics import auc, roc_curve
import common
import glob
import numpy as np
import os
import pandas as pd
import pylab
import time

def compute_Weighted_AUC(y_true, y_score, weight_distribution=np.arange(4, -1, -1.0)):
    """Compute the Weighted AUC score.
    
    :param y_true: true binary labels in range {0, 1}
    :type y_true: numpy array
    :param y_score: the probability estimates of the positive class
    :type y_score: numpy array
    :param weight_distribution: the weights of different areas
    :type weight_distribution: numpy array
    :return: the Weighted AUC score
    :rtype: float
    """

    def perform_interpolation(fpr, tpr, threshold_array):
        """Perform interpolation on the ROC curve.
        
        :param fpr: false positive rate
        :type fpr: numpy array
        :param tpr: true positive rate
        :type tpr: numpy array
        :param threshold_array: the thresholds where interpolation is needed
        :type threshold_array: numpy array
        :return: the interpolated fpr and tpr
        :rtype: tuple
        """

        for threshold in threshold_array:
            # Neglect the interpolation if the threshold is aleady in tpr
            if np.sum(tpr == threshold) > 0:
                continue

            # Find the index which meets tpr[previous_index] < threshold
            # and tpr[previous_index+1] > threshold
            previous_index = np.max(np.argwhere(tpr < threshold))
            following_index = previous_index + 1

            # Insert the interpolated data to tpr and fpr.
            # The interpolated data is generated by using linear interpolation.
            value = (threshold - tpr[previous_index]) * (fpr[following_index] - fpr[previous_index]) / \
                (tpr[following_index] - tpr[previous_index]) + fpr[previous_index]
            tpr = np.insert(tpr, following_index, threshold)
            fpr = np.insert(fpr, following_index, value)

        return (fpr, tpr)

    # Divide the range [0, 1] evenly
    weight_num = weight_distribution.shape[0]
    evenly_spaced_thresholds = np.linspace(0, 1, num=weight_num + 1)

    # Compute ROC curve and perform interpolation
    fpr, tpr, _ = roc_curve(y_true, y_score)
    fpr, tpr = perform_interpolation(fpr, tpr, evenly_spaced_thresholds[1:-1])

    # Plot ROC curve
    # pylab.figure()
    # pylab.plot(fpr, tpr)
    # pylab.axis("equal")
    # pylab.grid()
    # pylab.xlabel("False Positive Rate")
    # pylab.ylabel("True Positive Rate")
    # pylab.title("ROC Curve")
    # pylab.show()

    # Compute the areas
    area_array = np.zeros(weight_num)
    lowest_record_index = 0
    for area_index in range(weight_num):
        # Compute the highest index of the records within current area
        high_threshold = evenly_spaced_thresholds[area_index + 1]
        highest_record_index = np.sum(tpr <= high_threshold) - 1

        # Select the records within current area
        selected_fpr = fpr[lowest_record_index:highest_record_index + 1]
        selected_tpr = tpr[lowest_record_index:highest_record_index + 1]

        # Remove bias in True Positive Rate
        selected_tpr = selected_tpr - selected_tpr[0]

        # Extend the curve to the range [0, 1]
        selected_fpr = np.hstack(([0], selected_fpr, [1]))
        selected_tpr = np.hstack((selected_tpr[0], selected_tpr, selected_tpr[-1]))

        # Plot selected ROC curve
        # pylab.figure()
        # pylab.plot(selected_fpr, selected_tpr)
        # pylab.axis("equal")
        # pylab.grid()
        # pylab.xlabel("False Positive Rate")
        # pylab.ylabel("True Positive Rate")
        # pylab.title("Selected ROC Curve")
        # pylab.show()

        # Compute current area
        area_array[area_index] = auc(selected_fpr, selected_tpr, reorder=True)

        # Update lowest_record_index
        lowest_record_index = highest_record_index

    # Normalize weight distribution and return final score
    weight_distribution = weight_distribution / np.mean(weight_distribution)
    return np.sum(np.multiply(area_array, weight_distribution))

def perform_evaluation():
    """Perform evaluation on the submission files."""

    # Read GroundTruth file
    groundtruth_file_path = os.path.join(common.SUBMISSIONS_FOLDER_PATH, common.GROUNDTRUTH_FILE_NAME)
    groundtruth_file_content = pd.read_csv(groundtruth_file_path, skiprows=0).as_matrix()
    groundtruth_label = groundtruth_file_content[:, 1]
    groundtruth_label = groundtruth_label.astype(np.float64)

    # List all csv files in current folder and evaluate them
    submission_file_path_list = glob.glob(os.path.join(common.SUBMISSIONS_FOLDER_PATH, "*.csv"))
    for submission_file_path in submission_file_path_list:
        if submission_file_path == groundtruth_file_path:
            continue

        # Read current submission file
        submission_file_content = pd.read_csv(submission_file_path, skiprows=0).as_matrix()
        submission_label = submission_file_content[:, 1]

        # Compute Weighted AUC of current submission file
        score = compute_Weighted_AUC(groundtruth_label, submission_label)

        print("{} achieved {:.4f}.".format(os.path.basename(submission_file_path), score))

def combine_submissions(submission_file_name_rule_list):
    """Combine submissions.
    
    :param submission_file_name_rule_list: the file name rules of the submissions
    :type submission_file_name_rule_list: list
    :return: the new submission file will be created
    :rtype: None
    """

    def get_ranks(input_array):
        """Get the ranks of the elements in an array.
        
        :param input_array: input array
        :type input_array: numpy array
        :return: the ranks of the elements
        :rtype: numpy array
        """

        order = input_array.argsort()
        ranks = order.argsort()
        return ranks

    # Get the list of submission files based on the rules
    submission_file_path_list = []
    for submission_file_name_rule in submission_file_name_rule_list:
        current_submission_file_path_list = glob.glob(os.path.join(\
                                                                   common.SUBMISSIONS_FOLDER_PATH, \
                                                                   submission_file_name_rule))
        for current_submission_file_path in current_submission_file_path_list:
            if current_submission_file_path not in submission_file_path_list:
                submission_file_path_list.append(current_submission_file_path)

    # Read submission files
    submission_ranks_list = []
    for submission_file_path in submission_file_path_list:
        submission_file_content = pd.read_csv(submission_file_path, skiprows=0).as_matrix()
        submission_label = submission_file_content[:, 1]
        submission_ranks = get_ranks(submission_label)
        submission_ranks_list.append(submission_ranks)
    submission_ranks_array = np.array(submission_ranks_list)

    # Generate mean submission file
    mean_submission = np.mean(submission_ranks_array, axis=0)
    mean_submission_file_content = pd.DataFrame({\
                                                 "Id": np.arange(mean_submission.shape[0]), \
                                                 "Prediction": mean_submission})
    mean_submission_file_content.to_csv(os.path.join(\
                                        common.SUBMISSIONS_FOLDER_PATH, "mean_submission_" + str(int(time.time())) + ".csv"), \
                                        index=False, header=True)

    # Generate median submission file
    median_submission = np.median(submission_ranks_array, axis=0)
    median_submission_file_content = pd.DataFrame({\
                                                   "Id": np.arange(median_submission.shape[0]), \
                                                   "Prediction": median_submission})
    median_submission_file_content.to_csv(os.path.join(\
                                        common.SUBMISSIONS_FOLDER_PATH, "median_submission_" + str(int(time.time())) + ".csv"), \
                                        index=False, header=True)

if __name__ == "__main__":
    # combine_submissions(["prediction_open_face_open_face_keras*.csv"])
    # combine_submissions(["prediction_bbox_vgg_face_keras*.csv"])
    perform_evaluation()
<commit_msg>Sort the submission_file_path_list.<commit_after>from sklearn.metrics import auc, roc_curve
import common
import glob
import numpy as np
import os
import pandas as pd
import pylab
import time

def compute_Weighted_AUC(y_true, y_score, weight_distribution=np.arange(4, -1, -1.0)):
    """Compute the Weighted AUC score.
    
    :param y_true: true binary labels in range {0, 1}
    :type y_true: numpy array
    :param y_score: the probability estimates of the positive class
    :type y_score: numpy array
    :param weight_distribution: the weights of different areas
    :type weight_distribution: numpy array
    :return: the Weighted AUC score
    :rtype: float
    """

    def perform_interpolation(fpr, tpr, threshold_array):
        """Perform interpolation on the ROC curve.
        
        :param fpr: false positive rate
        :type fpr: numpy array
        :param tpr: true positive rate
        :type tpr: numpy array
        :param threshold_array: the thresholds where interpolation is needed
        :type threshold_array: numpy array
        :return: the interpolated fpr and tpr
        :rtype: tuple
        """

        for threshold in threshold_array:
            # Neglect the interpolation if the threshold is aleady in tpr
            if np.sum(tpr == threshold) > 0:
                continue

            # Find the index which meets tpr[previous_index] < threshold
            # and tpr[previous_index+1] > threshold
            previous_index = np.max(np.argwhere(tpr < threshold))
            following_index = previous_index + 1

            # Insert the interpolated data to tpr and fpr.
            # The interpolated data is generated by using linear interpolation.
            value = (threshold - tpr[previous_index]) * (fpr[following_index] - fpr[previous_index]) / \
                (tpr[following_index] - tpr[previous_index]) + fpr[previous_index]
            tpr = np.insert(tpr, following_index, threshold)
            fpr = np.insert(fpr, following_index, value)

        return (fpr, tpr)

    # Divide the range [0, 1] evenly
    weight_num = weight_distribution.shape[0]
    evenly_spaced_thresholds = np.linspace(0, 1, num=weight_num + 1)

    # Compute ROC curve and perform interpolation
    fpr, tpr, _ = roc_curve(y_true, y_score)
    fpr, tpr = perform_interpolation(fpr, tpr, evenly_spaced_thresholds[1:-1])

    # Plot ROC curve
    # pylab.figure()
    # pylab.plot(fpr, tpr)
    # pylab.axis("equal")
    # pylab.grid()
    # pylab.xlabel("False Positive Rate")
    # pylab.ylabel("True Positive Rate")
    # pylab.title("ROC Curve")
    # pylab.show()

    # Compute the areas
    area_array = np.zeros(weight_num)
    lowest_record_index = 0
    for area_index in range(weight_num):
        # Compute the highest index of the records within current area
        high_threshold = evenly_spaced_thresholds[area_index + 1]
        highest_record_index = np.sum(tpr <= high_threshold) - 1

        # Select the records within current area
        selected_fpr = fpr[lowest_record_index:highest_record_index + 1]
        selected_tpr = tpr[lowest_record_index:highest_record_index + 1]

        # Remove bias in True Positive Rate
        selected_tpr = selected_tpr - selected_tpr[0]

        # Extend the curve to the range [0, 1]
        selected_fpr = np.hstack(([0], selected_fpr, [1]))
        selected_tpr = np.hstack((selected_tpr[0], selected_tpr, selected_tpr[-1]))

        # Plot selected ROC curve
        # pylab.figure()
        # pylab.plot(selected_fpr, selected_tpr)
        # pylab.axis("equal")
        # pylab.grid()
        # pylab.xlabel("False Positive Rate")
        # pylab.ylabel("True Positive Rate")
        # pylab.title("Selected ROC Curve")
        # pylab.show()

        # Compute current area
        area_array[area_index] = auc(selected_fpr, selected_tpr, reorder=True)

        # Update lowest_record_index
        lowest_record_index = highest_record_index

    # Normalize weight distribution and return final score
    weight_distribution = weight_distribution / np.mean(weight_distribution)
    return np.sum(np.multiply(area_array, weight_distribution))

def perform_evaluation():
    """Perform evaluation on the submission files."""

    # Read GroundTruth file
    groundtruth_file_path = os.path.join(common.SUBMISSIONS_FOLDER_PATH, common.GROUNDTRUTH_FILE_NAME)
    groundtruth_file_content = pd.read_csv(groundtruth_file_path, skiprows=0).as_matrix()
    groundtruth_label = groundtruth_file_content[:, 1]
    groundtruth_label = groundtruth_label.astype(np.float64)

    # List all csv files in current folder and evaluate them
    submission_file_path_list = glob.glob(os.path.join(common.SUBMISSIONS_FOLDER_PATH, "*.csv"))
    submission_file_path_list = sorted(submission_file_path_list)
    for submission_file_path in submission_file_path_list:
        if submission_file_path == groundtruth_file_path:
            continue

        # Read current submission file
        submission_file_content = pd.read_csv(submission_file_path, skiprows=0).as_matrix()
        submission_label = submission_file_content[:, 1]

        # Compute Weighted AUC of current submission file
        score = compute_Weighted_AUC(groundtruth_label, submission_label)

        print("{} achieved {:.4f}.".format(os.path.basename(submission_file_path), score))

def combine_submissions(submission_file_name_rule_list):
    """Combine submissions.
    
    :param submission_file_name_rule_list: the file name rules of the submissions
    :type submission_file_name_rule_list: list
    :return: the new submission file will be created
    :rtype: None
    """

    def get_ranks(input_array):
        """Get the ranks of the elements in an array.
        
        :param input_array: input array
        :type input_array: numpy array
        :return: the ranks of the elements
        :rtype: numpy array
        """

        order = input_array.argsort()
        ranks = order.argsort()
        return ranks

    # Get the list of submission files based on the rules
    submission_file_path_list = []
    for submission_file_name_rule in submission_file_name_rule_list:
        current_submission_file_path_list = glob.glob(os.path.join(\
                                                                   common.SUBMISSIONS_FOLDER_PATH, \
                                                                   submission_file_name_rule))
        for current_submission_file_path in current_submission_file_path_list:
            if current_submission_file_path not in submission_file_path_list:
                submission_file_path_list.append(current_submission_file_path)

    # Read submission files
    submission_ranks_list = []
    for submission_file_path in submission_file_path_list:
        submission_file_content = pd.read_csv(submission_file_path, skiprows=0).as_matrix()
        submission_label = submission_file_content[:, 1]
        submission_ranks = get_ranks(submission_label)
        submission_ranks_list.append(submission_ranks)
    submission_ranks_array = np.array(submission_ranks_list)

    # Generate mean submission file
    mean_submission = np.mean(submission_ranks_array, axis=0)
    mean_submission_file_content = pd.DataFrame({\
                                                 "Id": np.arange(mean_submission.shape[0]), \
                                                 "Prediction": mean_submission})
    mean_submission_file_content.to_csv(os.path.join(\
                                        common.SUBMISSIONS_FOLDER_PATH, "mean_submission_" + str(int(time.time())) + ".csv"), \
                                        index=False, header=True)

    # Generate median submission file
    median_submission = np.median(submission_ranks_array, axis=0)
    median_submission_file_content = pd.DataFrame({\
                                                   "Id": np.arange(median_submission.shape[0]), \
                                                   "Prediction": median_submission})
    median_submission_file_content.to_csv(os.path.join(\
                                        common.SUBMISSIONS_FOLDER_PATH, "median_submission_" + str(int(time.time())) + ".csv"), \
                                        index=False, header=True)

if __name__ == "__main__":
    # combine_submissions(["prediction_open_face_open_face_keras*.csv"])
    # combine_submissions(["prediction_bbox_vgg_face_keras*.csv"])
    perform_evaluation()

{'fraction_non_alphanumeric': 0.0813829155280979, 'fraction_numerical': 0.004989901390043959, 'mean_word_length': 2.6526361466695594, 'pattern_counts': {'":': 10, '<': 9, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 32, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1144566', 'n_tokens_mistral': 5030, 'n_tokens_neox': 4796, 'n_words': 1146}