<commit_before>import numpy as np

import sys

from sklearn.metrics import precision_score
from sklearn.metrics import recall_score
from sklearn.metrics import accuracy_score

from arch.api import session
from federatedml.feature.instance import Instance
from federatedml.feature.sparse_vector import SparseVector

import copy

import pandas as pd


ROUND_NUM = 6

def neg_pos_count(labels: np.ndarray, pos_label: int):
    pos_num = ((labels == pos_label) + 0).sum()
    neg_num = len(labels) - pos_num
    return pos_num, neg_num


def sort_score_and_label(labels: np.ndarray, pred_scores: np.ndarray):
    labels = np.array(labels)
    pred_scores = np.array(pred_scores)

    sort_idx = np.flip(pred_scores.argsort())
    sorted_labels = labels[sort_idx]
    sorted_scores = pred_scores[sort_idx]

    return sorted_labels, sorted_scores


def map_ndarray_to_dtable(arr, partitions=10):

    instances = [Instance(features=SparseVector(indices=[0], data=[arr[i]])) for i in arr]
    dt = session.parallelize(instances, partition=partitions)
    dt.schema['header'] = ['scores']

    return dt


class ConfusionMatrix(object):

    @staticmethod
    def compute(sorted_labels: list, sorted_pred_scores: list, score_thresholds: list, ret: list):

        for ret_type in ret:
            assert ret_type in ['tp', 'tn', 'fp', 'fn']

        sorted_labels = np.array(sorted_labels)
        sorted_scores = sorted_pred_scores

        score_thresholds = np.array([score_thresholds]).transpose()
        pred_labels = (sorted_scores > score_thresholds) + 0

        ret_dict = {}
        if 'tp' in ret or 'tn' in ret:
            match_arr = (pred_labels + sorted_labels)
            if 'tp' in ret:
                tp_num = (match_arr == 2).sum(axis=-1)
                ret_dict['tp'] = tp_num
            if 'tn' in ret:
                tn_num = (match_arr == 0).sum(axis=-1)
                ret_dict['tn'] = tn_num

        if 'fp' in ret or 'fn' in ret:
            match_arr = (sorted_labels - pred_labels)
            if 'fp' in ret:
                fp_num = (match_arr == -1).sum(axis=-1)
                ret_dict['fp'] = fp_num
            if 'fn' in ret:
                fn_num = (match_arr == 1).sum(axis=-1)
                ret_dict['fn'] = fn_num

        return ret_dict


class ThresholdCutter(object):

    @staticmethod
    def cut_by_step(sorted_scores, steps=0.01):
        assert isinstance(steps, float) and (0 < steps < 1)
        thresholds = list(set(sorted_scores))
        thresholds, cuts = ThresholdCutter.__filt_threshold(thresholds, 0.01)
        score_threshold = thresholds

        return score_threshold, cuts

    @staticmethod
    def cut_by_index(sorted_scores):
        cuts = np.array([c / 100 for c in range(100)])
        data_size = len(sorted_scores)
        indexs = [int(data_size * cut) for cut in cuts]
        score_threshold = [sorted_scores[idx] for idx in indexs]
        return score_threshold, cuts

    @staticmethod
    def __filt_threshold(thresholds, step):
        cuts = list(map(float, np.arange(0, 1, step)))
        size = len(list(thresholds))
        thresholds.sort(reverse=True)
        index_list = [int(size * cut) for cut in cuts]
        new_thresholds = [thresholds[idx] for idx in index_list]

        return new_thresholds, cuts

    @staticmethod
    def cut_by_quantile(scores, quantile_list=None, interpolation='nearest', remove_duplicate=True):

        if quantile_list is None:  # default is 20 intervals
            quantile_list = [round(i * 0.05, 3) for i in range(20)] + [1.0]
        quantile_val = np.quantile(scores, quantile_list, interpolation=interpolation)
        if remove_duplicate:
            quantile_val = sorted(list(set(quantile_val)))
        else:
            quantile_val = sorted(list(quantile_val))

        if len(quantile_val) == 1:
            quantile_val = [np.min(scores), np.max(scores)]

        return quantile_val


class KS(object):

    @staticmethod
    def compute(labels, pred_scores, pos_label=1):

        sorted_labels, sorted_scores = sort_score_and_label(labels, pred_scores)

        score_threshold, cuts = ThresholdCutter.cut_by_index(sorted_scores)

        confusion_mat = ConfusionMatrix.compute(sorted_labels, sorted_scores, score_threshold, ret=['tp', 'fp'])

        pos_num, neg_num = neg_pos_count(sorted_labels, pos_label=pos_label)<commit_msg>add sample count examination in ks<commit_after>import numpy as np

import sys

from sklearn.metrics import precision_score
from sklearn.metrics import recall_score
from sklearn.metrics import accuracy_score

from arch.api import session
from federatedml.feature.instance import Instance
from federatedml.feature.sparse_vector import SparseVector

import copy

import pandas as pd


ROUND_NUM = 6


def neg_pos_count(labels: np.ndarray, pos_label: int):
    pos_num = ((labels == pos_label) + 0).sum()
    neg_num = len(labels) - pos_num
    return pos_num, neg_num


def sort_score_and_label(labels: np.ndarray, pred_scores: np.ndarray):
    labels = np.array(labels)
    pred_scores = np.array(pred_scores)

    sort_idx = np.flip(pred_scores.argsort())
    sorted_labels = labels[sort_idx]
    sorted_scores = pred_scores[sort_idx]

    return sorted_labels, sorted_scores


def map_ndarray_to_dtable(arr, partitions=10):

    instances = [Instance(features=SparseVector(indices=[0], data=[arr[i]])) for i in arr]
    dt = session.parallelize(instances, partition=partitions)
    dt.schema['header'] = ['scores']

    return dt


class ConfusionMatrix(object):

    @staticmethod
    def compute(sorted_labels: list, sorted_pred_scores: list, score_thresholds: list, ret: list):

        for ret_type in ret:
            assert ret_type in ['tp', 'tn', 'fp', 'fn']

        sorted_labels = np.array(sorted_labels)
        sorted_scores = sorted_pred_scores

        score_thresholds = np.array([score_thresholds]).transpose()
        pred_labels = (sorted_scores > score_thresholds) + 0

        ret_dict = {}
        if 'tp' in ret or 'tn' in ret:
            match_arr = (pred_labels + sorted_labels)
            if 'tp' in ret:
                tp_num = (match_arr == 2).sum(axis=-1)
                ret_dict['tp'] = tp_num
            if 'tn' in ret:
                tn_num = (match_arr == 0).sum(axis=-1)
                ret_dict['tn'] = tn_num

        if 'fp' in ret or 'fn' in ret:
            match_arr = (sorted_labels - pred_labels)
            if 'fp' in ret:
                fp_num = (match_arr == -1).sum(axis=-1)
                ret_dict['fp'] = fp_num
            if 'fn' in ret:
                fn_num = (match_arr == 1).sum(axis=-1)
                ret_dict['fn'] = fn_num

        return ret_dict


class ThresholdCutter(object):

    @staticmethod
    def cut_by_step(sorted_scores, steps=0.01):
        assert isinstance(steps, float) and (0 < steps < 1)
        thresholds = list(set(sorted_scores))
        thresholds, cuts = ThresholdCutter.__filt_threshold(thresholds, 0.01)
        score_threshold = thresholds

        return score_threshold, cuts

    @staticmethod
    def cut_by_index(sorted_scores):
        cuts = np.array([c / 100 for c in range(100)])
        data_size = len(sorted_scores)
        indexs = [int(data_size * cut) for cut in cuts]
        score_threshold = [sorted_scores[idx] for idx in indexs]
        return score_threshold, cuts

    @staticmethod
    def __filt_threshold(thresholds, step):
        cuts = list(map(float, np.arange(0, 1, step)))
        size = len(list(thresholds))
        thresholds.sort(reverse=True)
        index_list = [int(size * cut) for cut in cuts]
        new_thresholds = [thresholds[idx] for idx in index_list]

        return new_thresholds, cuts

    @staticmethod
    def cut_by_quantile(scores, quantile_list=None, interpolation='nearest', remove_duplicate=True):

        if quantile_list is None:  # default is 20 intervals
            quantile_list = [round(i * 0.05, 3) for i in range(20)] + [1.0]
        quantile_val = np.quantile(scores, quantile_list, interpolation=interpolation)
        if remove_duplicate:
            quantile_val = sorted(list(set(quantile_val)))
        else:
            quantile_val = sorted(list(quantile_val))

        if len(quantile_val) == 1:
            quantile_val = [np.min(scores), np.max(scores)]

        return quantile_val


class KS(object):

    @staticmethod
    def compute(labels, pred_scores, pos_label=1):

        sorted_labels, sorted_scores = sort_score_and_label(labels, pred_scores)

        score_threshold, cuts = ThresholdCutter.cut_by_index(sorted_scores)

        confusion_mat = ConfusionMatrix.compute(sorted_labels, sorted_scores, score_threshold, ret=['tp', 'fp'])

        pos_num, neg_num = neg_pos_count(sorted_labels, pos_label=pos_label)

        assert pos_num > 0 and neg_num > 0, "error when computing KS metric, pos sample number and neg sample number" \
                                            "must be larger than 0"
{'fraction_non_alphanumeric': 0.10023413981491805, 'fraction_numerical': 0.009700078046604972, 'mean_word_length': 2.841541755888651, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '369459', 'n_tokens_mistral': 3006, 'n_tokens_neox': 2796, 'n_words': 683}