<commit_before>import pandas as pd
import numpy as np


"""
Use this toolbox either as an imported module, or in e.g. ipython with the magic %paste command 
to import w.e. is in you clipboard; that could be the jaccard function or one of the other tools.

Note to self:
Itemset finder with regex

^\s+([01]\s+){1}1\s+([01]\s+){4}1
"""



def jaccard(a,b):
    _11 = 0
    _not00 = 0
    for x,y in zip(a,b):
        if x and y:
            _11 += 1
            _not00 += 1
        elif x or y:
            _not00 += 1
    return float(_11)/float(_not00)

"""
Use of jaccard:
for i1 in range(5):
    for i2 in range(5):
        if i1 >= i2:
            continue
        print i1+1, i2+1, 1 - jaccard(_ns[i1], _ns[i2])
"""


def gini(vec):
    # It is 1 minus the sum over i of p_i^2, where p_i is the fraction of records belonging to class i.
    asseries = pd.Series(vec)
    valc = asseries.value_counts()
    normed = valc / len(vec)
    powered = normed * normed
    summed = powered.sum()
    return (1 - summed)

def classification_error(vec):
    return 1 - float(pd.Series(vec).value_counts()[0])/len(vec)

def purity_gain(parent, children, measure_method=gini):
    """
    Usage: purity_gain([1,0,1,0], [[0,0],[1,1]], gini)
    """
    # break early:
    children[0][1] # breaks if you're e.g. just passing a 1d list of chars
    #It is the impurity of the parent minus the sum over i of 
    #   (the number of records associated with the child node i divided by the total number of records in the parent node, 
    #   multiplied by the impurity measure of the child node i)
    pval = measure_method(parent)
    pl = float(len(parent))
    chvals = [measure_method(x) * len(x) / pl for x in children]
    
    return pval - sum(chvals)


def least_square(A, y):
    """
    Intercept is placed first
    """
    A = np.vstack([np.ones(len(A)), A]).T
    return np.linalg.lstsq(A, y)[0]


def standardise(vec, ddof=0):
    """
    subtract mean, then divide by standard deviation
    ddof is used as N - ddof in the divisor in std
    """
    vm = np.mean(vec)
    vs = np.std(vec, ddof=ddof)
    return [(x - vm)/vs for x in vec]

def pca_accountability(singular_values):
    """
    Answers the eternal question: How much variability does the nth principal component account for?
    Returns the fraction of variability accounting for each of the elements in singular_values, order preserved.
    """

    squared = [x*x for x in singular_values]
    svsum = float(sum(squared))
    ratio = [x / svsum for x in squared]
    return ratio


def classification_stats(TP, TN, FP, FN):
    count = TP + TN + FP + FN
    error_rate = float(FP + FN) / count
    TPR = TP / float(TP + FN)
    FPR = FP / float(TN + FP)
    FNR = FN / float(TP + FN)
    TNR = TN / float(TN + FP)
    sensitivity = TPR
    specificity = TN
    precision = TP / float(TP + FP)
    recall = TP / float(TP + FN)
    F1 = 2 * TP / float(2 * TP + FP + FN)

    output = {
        'count': count,
        'error_rate': error_rate,
        'TPR': TPR,
        'FPR': FPR,
        'FNR': FNR,
        'TNR': TNR,
        'sensitivity': sensitivity,
        'specificity': specificity,
        'precision': precision,
        'recall': recall,
        'F1': F1
    }

    return output


# Wrapper methods

def mean(vals):
    return pd.Series(vals).mean()

def median(vals):<commit_msg>Added itemset generator with support_min requirement<commit_after>import pandas as pd
import numpy as np
import itertools as IT

"""
Use this toolbox either as an imported module, or in e.g. ipython with the magic %paste command 
to import w.e. is in you clipboard; that could be the jaccard function or one of the other tools.

Note to self:
Itemset finder with regex

^\s+([01]\s+){1}1\s+([01]\s+){4}1
"""



def jaccard(a,b):
    _11 = 0
    _not00 = 0
    for x,y in zip(a,b):
        if x and y:
            _11 += 1
            _not00 += 1
        elif x or y:
            _not00 += 1
    return float(_11)/float(_not00)

"""
Use of jaccard:
for i1 in range(5):
    for i2 in range(5):
        if i1 >= i2:
            continue
        print i1+1, i2+1, 1 - jaccard(_ns[i1], _ns[i2])
"""


def gini(vec):
    # It is 1 minus the sum over i of p_i^2, where p_i is the fraction of records belonging to class i.
    asseries = pd.Series(vec)
    valc = asseries.value_counts()
    normed = valc / len(vec)
    powered = normed * normed
    summed = powered.sum()
    return (1 - summed)

def classification_error(vec):
    return 1 - float(pd.Series(vec).value_counts()[0])/len(vec)

def purity_gain(parent, children, measure_method=gini):
    """
    Usage: purity_gain([1,0,1,0], [[0,0],[1,1]], gini)
    """
    # break early:
    children[0][1] # breaks if you're e.g. just passing a 1d list of chars
    #It is the impurity of the parent minus the sum over i of 
    #   (the number of records associated with the child node i divided by the total number of records in the parent node, 
    #   multiplied by the impurity measure of the child node i)
    pval = measure_method(parent)
    pl = float(len(parent))
    chvals = [measure_method(x) * len(x) / pl for x in children]
    
    return pval - sum(chvals)


def least_square(A, y):
    """
    Intercept is placed first
    """
    A = np.vstack([np.ones(len(A)), A]).T
    return np.linalg.lstsq(A, y)[0]


def standardise(vec, ddof=0):
    """
    subtract mean, then divide by standard deviation
    ddof is used as N - ddof in the divisor in std
    """
    vm = np.mean(vec)
    vs = np.std(vec, ddof=ddof)
    return [(x - vm)/vs for x in vec]

def pca_accountability(singular_values):
    """
    Answers the eternal question: How much variability does the nth principal component account for?
    Returns the fraction of variability accounting for each of the elements in singular_values, order preserved.
    """

    squared = [x*x for x in singular_values]
    svsum = float(sum(squared))
    ratio = [x / svsum for x in squared]
    return ratio


def classification_stats(TP, TN, FP, FN):
    count = TP + TN + FP + FN
    error_rate = float(FP + FN) / count
    TPR = TP / float(TP + FN)
    FPR = FP / float(TN + FP)
    FNR = FN / float(TP + FN)
    TNR = TN / float(TN + FP)
    sensitivity = TPR
    specificity = TN
    precision = TP / float(TP + FP)
    recall = TP / float(TP + FN)
    F1 = 2 * TP / float(2 * TP + FP + FN)

    output = {
        'count': count,
        'error_rate': error_rate,
        'TPR': TPR,
        'FPR': FPR,
        'FNR': FNR,
        'TNR': TNR,
        'sensitivity': sensitivity,
        'specificity': specificity,
        'precision': precision,
        'recall': recall,
        'F1': F1
    }

    return output


def support(itemset):
    """
    Returns the support value for the given itemset
    itemset is a pandas dataframe with one row per basket, and one column per item
    """

    # Get the count of baskets where all the items are 1
    baskets = itemset.iloc[:,0].copy()
    for col in itemset.columns[1:]:
        baskets = baskets & itemset[col]

    return baskets.sum() / float(len(baskets))


def itemsets(df, support_min):
    """
    df is a pandas dataframe, with each row being a basket, and each column being an item
    You can really really really benefit from writing the table in a text editor, copying it,
    and using pd.from_clipboard().
    """
    itemsets = []
    n = len(df)
    for itsetSize in np.arange(1, len(df.columns) + 1): # Start with 1-itemsets, keep going till n_attributes-itemsets
        for combination in IT.combinations(df.columns, itsetSize):
            sup = support(df[list(combination)])
            if sup > support_min:
                itemsets.append(set(combination))

    return itemsets



# Wrapper methods

def mean(vals):
    return pd.Series(vals).mean()

def median(vals):
{'fraction_non_alphanumeric': 0.11467245562507981, 'fraction_numerical': 0.016856084791214403, 'mean_word_length': 2.497990174184904, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 6, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '170026', 'n_tokens_mistral': 2800, 'n_tokens_neox': 2607, 'n_words': 922}