<commit_before>import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
from . import condition

class ResultsProcessor:

    def __init__(self, betas, rescaled_betas, threshold=0.5):
        self.betas = betas
        self.rescaled_betas = rescaled_betas
        self.threshold = threshold

    def compute_combined_confidences(self):
        combined_confidences = pd.DataFrame(np.zeros((self.betas[0].shape)), index = self.betas[0].index, columns = self.betas[0].columns )
        for beta_resc in self.rescaled_betas:
            # this ranking code is especially wordy because the rank function only works in one dimension (col or row), so I had to flatten the matrix
            ranked_df =np.reshape(pd.DataFrame(np.ndarray.flatten(beta_resc.values)).rank( method = "average").values, self.rescaled_betas[0].shape)
            combined_confidences = combined_confidences + ranked_df

        min_element = min(combined_confidences.min())
        combined_confidences = (combined_confidences - min_element) / (len(self.betas) * combined_confidences.size - min_element)
        return combined_confidences

    def threshold_and_summarize(self):
        betas_sign = pd.DataFrame(np.zeros((self.betas[0].shape)), index = self.betas[0].index, columns = self.betas[0].columns )
        betas_non_zero = pd.DataFrame(np.zeros((self.betas[0].shape)), index = self.betas[0].index, columns = self.betas[0].columns )
        for beta in self.betas:
            betas_sign = betas_sign + np.sign(beta.values)
            betas_non_zero = betas_non_zero + np.absolute(np.sign(beta.values))

        # we only care about interactions that are present in more than th (fraction) bootstraps
        index_vector = np.where( betas_non_zero > len(self.betas) * self.threshold)
        betas_stack = np.stack([b.values[index_vector] for b in self.betas])
        return betas_stack

    def calculate_aupr(self, combined_confidences, gold_standard):
        candidates = np.where( combined_confidences > 0)
        # filter gold standard
        gold_standard_filtered = gold_standard_filtered[combined_confidences.columns]<commit_msg>betas stack should be a pandas data frame so it can be written to disk with rows and columns labeled<commit_after>import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
from . import condition

class ResultsProcessor:

    def __init__(self, betas, rescaled_betas, threshold=0.5):
        self.betas = betas
        self.rescaled_betas = rescaled_betas
        self.threshold = threshold

    def compute_combined_confidences(self):
        combined_confidences = pd.DataFrame(np.zeros((self.betas[0].shape)), index = self.betas[0].index, columns = self.betas[0].columns )
        for beta_resc in self.rescaled_betas:
            # this ranking code is especially wordy because the rank function only works in one dimension (col or row), so I had to flatten the matrix
            ranked_df =np.reshape(pd.DataFrame(np.ndarray.flatten(beta_resc.values)).rank( method = "average").values, self.rescaled_betas[0].shape)
            combined_confidences = combined_confidences + ranked_df

        min_element = min(combined_confidences.min())
        combined_confidences = (combined_confidences - min_element) / (len(self.betas) * combined_confidences.size - min_element)
        return combined_confidences

    def threshold_and_summarize(self):
        betas_sign = pd.DataFrame(np.zeros((self.betas[0].shape)), index = self.betas[0].index, columns = self.betas[0].columns )
        betas_non_zero = pd.DataFrame(np.zeros((self.betas[0].shape)), index = self.betas[0].index, columns = self.betas[0].columns )
        for beta in self.betas:
            betas_sign = betas_sign + np.sign(beta.values)
            betas_non_zero = betas_non_zero + np.absolute(np.sign(beta.values))

        # we only care about interactions that are present in more than th (fraction) bootstraps
        index_vector = np.where( betas_non_zero > len(self.betas) * self.threshold)
        betas_stack = pd.DataFrame(np.stack([b.values[index_vector] for b in self.betas]), index = self.betas[0].index, columns = self.betas[0].columns )
        return betas_stack

    def calculate_aupr(self, combined_confidences, gold_standard):
        candidates = np.where( combined_confidences > 0)
        # filter gold standard
        gold_standard_filtered = gold_standard_filtered[combined_confidences.columns]
{'fraction_non_alphanumeric': 0.09657811796488068, 'fraction_numerical': 0.006303466906798739, 'mean_word_length': 3.5522540983606556, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '911433', 'n_tokens_mistral': 1392, 'n_tokens_neox': 1356, 'n_words': 316}