<commit_before>import numpy as np
import pandas as pd
from collections import Counter
from itertools import combinations


def is_categorical(x):
    '''
    INPUT
         - single data point x
    OUTPUT
         - boolean
    returns true if x is categorical else false
    '''
    return isinstance(x, str) or isinstance(x, bool) or isinstance(x, unicode)


def check_if_categorical(attribute, df):
    '''
    INPUT:
         - attribute: the feature inside the dataframe to check
         - df: the DataFrame itself
    OUTPUT:
         - boolean
    Returns True if feature in df is categorical else False
    '''
    check_if_categorical = np.vectorize(is_categorical)
    if np.mean(check_if_categorical(df[attribute].values)) == 1:
        return True
    return False


def entropy(y):
    '''
    INPUT:
        - y: 1d numpy array
    OUTPUT:
        - float

    Return the entropy of the array y.
    '''
    unique = set(y)
    count = Counter(y)
    ent = 0
    for val in unique:
        p = count[val]/float(len(y))
        ent += p * np.log2(p)
    return -1 * ent


def gini(y):
    '''
    INPUT:
        - y: 1d numpy array
    OUTPUT:
        - float

    Return the gini impurity of the array y.
    '''
    unique = np.unique(y)
    count = Counter(y)
    ent = 0
    for val in unique:
        p = count[val]/float(len(y))
        ent += p**2
    return 1 - ent


def make_split(X, y, split_value):
    '''
    INPUT:
        - X: 2d numpy array
        - y: 1d numpy array
        - split_value: int/float/bool/str (value of feature)
    OUTPUT:
        - X1: 2d numpy array (feature matrix for subset 1)
        - y1: 1d numpy array (labels for subset 1)
        - X2: 2d numpy array (feature matrix for subset 2)
        - y2: 1d numpy array (labels for subset 2)

    Return the two subsets of the dataset achieved by the given feature and
    value to split on.

    Call the method like this:
    >>> X1, y1, X2, y2 = make_split(X, y, split_value)

    X1, y1 is a subset of the data.
    X2, y2 is the other subset of the data.
    '''
    X1 = X[X <= split_value]
    y1 = y[X <= split_value]
    X2 = X[X > split_value]
    y2 = y[X > split_value]
    return X1, y1, X2, y2


def information_gain(y, y1, y2, impurity_criterion):
    '''
    INPUT:
        - y: 1d numpy array
        - y1: 1d numpy array (labels for subset 1)
        - y2: 1d numpy array (labels for subset 2)
    OUTPUT:
        - float
    Return the information gain of making the given split.
    '''
    return impurity_criterion(y) - \
        (float(len(y1))/len(y) * impurity_criterion(y1) +
            float(len(y2))/len(y) * impurity_criterion(y2))


def multiple_information_gain(y, y_list, impurity_criterion):
    '''
    INPUT:
        - y: 1d numpy array
        - y_list: list of y values [y1, y2, y3]
        - impurity_criterion: either gini or entropy
    OUTPUT:
        - float
    Return the information gain of making the given split.
    '''
    aggregate_entropy = 0
    for y_vals in y_list:
        aggregate_entropy += float(len(y_vals))/len(y) * \
            impurity_criterion(y_vals)
    return impurity_criterion(y) - aggregate_entropy


def determine_optimal_continuous_split_values(attribute, df, y):
    '''
    INPUT
         - attribute: str, feature to check
         - df: pandas dataframe of features
         - y: 1d array, target
    OUTPUT
         - max_split: tuple of best values to split on
         - info_gain_array: numpy array of all information gains
         - possible_splits: list of all possible split values
    Returns tuple of split values that optimize information gain (min 1 max 3)
    '''
    attribute_value_array = df[attribute].values
    split_values = np.unique(sorted(attribute_value_array))[:-1]
    info_gain_list = []
    info_gain = 0
    possible_splits = list(combinations(split_values, 1)) + \
        list(combinations(split_values, 2)) + \
        list(combinations(split_values, 3))
    for split in possible_splits:
        X_list, y_list = make_multiple_split(attribute_value_array, y, split)
        info_gain = multiple_information_gain(y, y_list, entropy)
        info_gain_list.append(info_gain)
        info_gain_array = np.array(info_gain_list)
        max_split = possible_splits[np.argmax(info_gain_array)]<commit_msg>added split_list function instead of importing it to avoid errors and will now include multiprocessing in finding the optimal split<commit_after>import numpy as np
import pandas as pd
from collections import Counter
from itertools import combinations
import multiprocessing as mp


def is_categorical(x):
    '''
    INPUT
         - single data point x
    OUTPUT
         - boolean
    returns true if x is categorical else false
    '''
    return isinstance(x, str) or isinstance(x, bool) or isinstance(x, unicode)


def check_if_categorical(attribute, df):
    '''
    INPUT:
         - attribute: the feature inside the dataframe to check
         - df: the DataFrame itself
    OUTPUT:
         - boolean
    Returns True if feature in df is categorical else False
    '''
    check_if_categorical = np.vectorize(is_categorical)
    if np.mean(check_if_categorical(df[attribute].values)) == 1:
        return True
    return False


def entropy(y):
    '''
    INPUT:
        - y: 1d numpy array
    OUTPUT:
        - float

    Return the entropy of the array y.
    '''
    unique = set(y)
    count = Counter(y)
    ent = 0
    for val in unique:
        p = count[val]/float(len(y))
        ent += p * np.log2(p)
    return -1 * ent


def gini(y):
    '''
    INPUT:
        - y: 1d numpy array
    OUTPUT:
        - float

    Return the gini impurity of the array y.
    '''
    unique = np.unique(y)
    count = Counter(y)
    ent = 0
    for val in unique:
        p = count[val]/float(len(y))
        ent += p**2
    return 1 - ent


def make_split(X, y, split_value):
    '''
    INPUT:
        - X: 2d numpy array
        - y: 1d numpy array
        - split_value: int/float/bool/str (value of feature)
    OUTPUT:
        - X1: 2d numpy array (feature matrix for subset 1)
        - y1: 1d numpy array (labels for subset 1)
        - X2: 2d numpy array (feature matrix for subset 2)
        - y2: 1d numpy array (labels for subset 2)

    Return the two subsets of the dataset achieved by the given feature and
    value to split on.

    Call the method like this:
    >>> X1, y1, X2, y2 = make_split(X, y, split_value)

    X1, y1 is a subset of the data.
    X2, y2 is the other subset of the data.
    '''
    X1 = X[X <= split_value]
    y1 = y[X <= split_value]
    X2 = X[X > split_value]
    y2 = y[X > split_value]
    return X1, y1, X2, y2


def information_gain(y, y1, y2, impurity_criterion):
    '''
    INPUT:
        - y: 1d numpy array
        - y1: 1d numpy array (labels for subset 1)
        - y2: 1d numpy array (labels for subset 2)
    OUTPUT:
        - float
    Return the information gain of making the given split.
    '''
    return impurity_criterion(y) - \
        (float(len(y1))/len(y) * impurity_criterion(y1) +
            float(len(y2))/len(y) * impurity_criterion(y2))


def multiple_information_gain(y, y_list, impurity_criterion):
    '''
    INPUT:
        - y: 1d numpy array
        - y_list: list of y values [y1, y2, y3]
        - impurity_criterion: either gini or entropy
    OUTPUT:
        - float
    Return the information gain of making the given split.
    '''
    aggregate_entropy = 0
    for y_vals in y_list:
        aggregate_entropy += float(len(y_vals))/len(y) * \
            impurity_criterion(y_vals)
    return impurity_criterion(y) - aggregate_entropy


def determine_optimal_continuous_split_values(attribute, df, y):
    '''
    INPUT
         - attribute: str, feature to check
         - df: pandas dataframe of features
         - y: 1d array, target
    OUTPUT
         - max_split: tuple of best values to split on
         - info_gain_array: numpy array of all information gains
         - possible_splits: list of all possible split values
    Returns tuple of split values that optimize information gain (min 1 max 3)
    '''
    attribute_value_array = df[attribute].values
    split_values = np.unique(sorted(attribute_value_array))[:-1]
    info_gain_list = []
    info_gain = 0
    possible_splits = list(combinations(split_values, 1)) + \
        list(combinations(split_values, 2)) + \
        list(combinations(split_values, 3))
    for split in possible_splits:
        X_list, y_list = make_multiple_split(attribute_value_array, y, split)
        info_gain = multiple_information_gain(y, y_list, entropy)
        info_gain_list.append(info_gain)
        info_gain_array = np.array(info_gain_list)
        max_split = possible_splits[np.argmax(info_gain_array)]
    return max_split


def split_list(doc_list, n_groups):
    '''
    INPUT
         - doc_list - is a list of documents to be split up
         - n_groups - is the number of groups to split the doc_list into
    OUTPUT
         - list
    Returns a list of len n_groups which seeks to evenly split up the original
    list into continuous sub_lists
    '''
    avg = len(doc_list) / float(n_groups)
    split_lists = []
    last = 0.0
    while last < len(doc_list):
        split_lists.append(doc_list[int(last):int(last + avg)])
        last += avg
    return split_lists
    

def determine_optimal_continuous_split_values_boosted(attribute, df, y):
    '''
    INPUT
         - attribute: str, feature to check
         - df: pandas dataframe of features
         - y: 1d array, target
    OUTPUT
         - max_split: tuple of best values to split on
         - info_gain_array: numpy array of all information gains
         - possible_splits: list of all possible split values
    Returns tuple of split values that optimize information gain (min 1 max 3)
    '''
    attribute_value_array = df[attribute].values
    split_values = np.unique(sorted(attribute_value_array))[:-1]
    info_gain_list = []
    info_gain = 0
    possible_splits = list(combinations(split_values, 1)) + \
        list(combinations(split_values, 2)) + \
        list(combinations(split_values, 3))
    for split in possible_splits:
        X_list, y_list = make_multiple_split(attribute_value_array, y, split)
        info_gain = multiple_information_gain(y, y_list, entropy)
        info_gain_list.append(info_gain)
        info_gain_array = np.array(info_gain_list)
        max_split = possible_splits[np.argmax(info_gain_array)]
{'fraction_non_alphanumeric': 0.08799694918486033, 'fraction_numerical': 0.013347316236056821, 'mean_word_length': 2.40032414910859, 'pattern_counts': {'":': 0, '<': 8, '<?xml version=': 0, '>': 13, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '791568', 'n_tokens_mistral': 3436, 'n_tokens_neox': 3248, 'n_words': 1047}