<commit_before>#! /usr/bin/python
import numpy as np
import random
from pprint import pprint

class RandomForest():

    def __init__(self):
        self.forest_training_data = None
        self.forest_classcat = None
        self.num_examples = 0
        self.feature_split_less = []
        self.feature_split_greater = []
        self.bag_index_count_less = []
        self.bag_index_count_greater = []

    def reset_bagging(self, num_trees, bag_size, examples, classcat):
        if num_trees <= 0:
            raise ValueError('num_trees must be greater than zero.')

        if bag_size <= 0:
            raise ValueError('bag_size must be greater than zero.')

        num_examples = examples.shape[0]
        if bag_size > num_examples:
            raise ValueError('bag_size is too large.')
        self.num_examples = num_examples

        if len(classcat) != num_examples:
            raise ValueError('Number of rows in examples does not match length of classcat.')

        self.num_trees = num_trees
        self.forest_training_data = examples
        self.forest_classcat = classcat

        num_features = examples.shape[1]
        half_num_examples = .5 * num_examples

        self.feature_split_less = []
        self.feature_split_greater = []
        self.bag_index_count_less = []
        self.bag_index_count_greater = []

        for _ in range(num_trees):
            bag_index_count = [0]*self.num_examples

            for _ in range(bag_size):
                bag_index_count[random.randint(0, self.num_examples-1)] += 1

            btmp1 = []
            btmp2 = []

            for i, j in enumerate(bag_index_count):
                if j > 0:
                    btmp1.extend([self.forest_training_data[i, :] for _ in range(j)])
                    btmp2.extend([self.forest_classcat[i] for _ in range(j)])

            bag = np.array(btmp1)
            bag_classcat = np.array(btmp2)

            col = random.randrange(0, num_features)

            if False:
                split_val = np.median(bag[:, col])
            else:
                ntotal = bag.shape[0]
                info_best = 0.
                split_val = None
                for tries in range(10):
                    split_val_try = np.random.uniform(np.amin(bag[:, col], axis=0), np.amax(bag[:, col], axis=0))
                    nleft = len(np.where(bag[:, col] < split_val_try)[0])
                    nright = ntotal - nleft
                    q1 = float(nleft)/ntotal
                    q2 = float(nright)/ntotal
                    info_try = -(q1*np.log2(q1) + q2*np.log2(q2))
                    if info_try > info_best:
                        info_best = info_try
                        split_val = split_val_try

            num_less = np.sum(bag_classcat[i] for i in range(bag_size) if bag[i, col] < split_val)

            if (num_less < half_num_examples):
                self.feature_split_greater.append((col, split_val))
                self.bag_index_count_greater.append(bag_index_count)
            else:
                self.feature_split_less.append((col, split_val))
                self.bag_index_count_less.append(bag_index_count)

    def transform(self, features_it):
        for fit in features_it:
            votes_less = np.sum(fit[i] < j for i, j in feature_split_less)
            votes_greater = np.sum(fit[i] >= j for i, j in feature_split_greater)

            all_votes = votes_less + votes_greater
            percentage_votes = all_votes/self.num_trees

            if percentage_votes >= 0.5:
                yield True, percentage_votes<commit_msg>Conversion to float.<commit_after>#! /usr/bin/python
import numpy as np
import random
from pprint import pprint

class RandomForest():

    def __init__(self):
        self.forest_training_data = None
        self.forest_classcat = None
        self.num_examples = 0
        self.feature_split_less = []
        self.feature_split_greater = []
        self.bag_index_count_less = []
        self.bag_index_count_greater = []

    def reset_bagging(self, num_trees, bag_size, examples, classcat):
        if num_trees <= 0:
            raise ValueError('num_trees must be greater than zero.')

        if bag_size <= 0:
            raise ValueError('bag_size must be greater than zero.')

        num_examples = examples.shape[0]
        if bag_size > num_examples:
            raise ValueError('bag_size is too large.')
        self.num_examples = num_examples

        if len(classcat) != num_examples:
            raise ValueError('Number of rows in examples does not match length of classcat.')

        self.num_trees = num_trees
        self.forest_training_data = examples
        self.forest_classcat = classcat

        num_features = examples.shape[1]
        half_num_examples = .5 * num_examples

        self.feature_split_less = []
        self.feature_split_greater = []
        self.bag_index_count_less = []
        self.bag_index_count_greater = []

        for _ in range(num_trees):
            bag_index_count = [0]*self.num_examples

            for _ in range(bag_size):
                bag_index_count[random.randint(0, self.num_examples-1)] += 1

            btmp1 = []
            btmp2 = []

            for i, j in enumerate(bag_index_count):
                if j > 0:
                    btmp1.extend([self.forest_training_data[i, :] for _ in range(j)])
                    btmp2.extend([self.forest_classcat[i] for _ in range(j)])

            bag = np.array(btmp1)
            bag_classcat = np.array(btmp2)

            col = random.randrange(0, num_features)

            if False:
                split_val = np.median(bag[:, col])
            else:
                ntotal = bag.shape[0]
                info_best = 0.
                split_val = None
                for tries in range(10):
                    split_val_try = np.random.uniform(np.amin(bag[:, col], axis=0), np.amax(bag[:, col], axis=0))
                    nleft = len(np.where(bag[:, col] < split_val_try)[0])
                    nright = ntotal - nleft
                    q1 = float(nleft)/ntotal
                    q2 = float(nright)/ntotal
                    info_try = -(q1*np.log2(q1) + q2*np.log2(q2))
                    if info_try > info_best:
                        info_best = info_try
                        split_val = split_val_try

            num_less = np.sum(bag_classcat[i] for i in range(bag_size) if bag[i, col] < split_val)

            if (num_less < half_num_examples):
                self.feature_split_greater.append((col, split_val))
                self.bag_index_count_greater.append(bag_index_count)
            else:
                self.feature_split_less.append((col, split_val))
                self.bag_index_count_less.append(bag_index_count)

    def transform(self, features_it):
        for fit in features_it:
            votes_less = np.sum(fit[i] < j for i, j in feature_split_less)
            votes_greater = np.sum(fit[i] >= j for i, j in feature_split_greater)

            all_votes = votes_less + votes_greater
            percentage_votes = float(all_votes)/self.num_trees

            if percentage_votes >= 0.5:
                yield True, percentage_votes
{'fraction_non_alphanumeric': 0.09296203934607924, 'fraction_numerical': 0.009697977279024661, 'mean_word_length': 1.8309803921568628, 'pattern_counts': {'":': 0, '<': 15, '<?xml version=': 0, '>': 13, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4079621', 'n_tokens_mistral': 2278, 'n_tokens_neox': 2125, 'n_words': 457}