<commit_before>import numpy
from sklearn.metrics import confusion_matrix

def load_data():
    train_labels = []
    with open('digitdata/traininglabels', 'rb') as f:
        for i, line in enumerate(f):
            train_labels.append(int(line))
    train_labels = numpy.array(train_labels, dtype=int)

    train_x = numpy.zeros((train_labels.shape[0] * 28 * 28))
    with open('digitdata/trainingimages', 'rb') as f:
        for i, line in enumerate(f):
            for j, char in enumerate(line.strip('\n')):
                if '+' == char or '#' == char:
                    train_x[i * 28 + j] = 1

    train_x = train_x.reshape((train_labels.shape[0], 28 * 28))
    new_train_x = numpy.zeros((train_labels.shape[0], 27 * 27))
    for i in xrange(0, 27 * 27, 2):
        if i % 28 != 0 and i / 28 % 2 == 0:
            new_train_x[:, i] = train_x[:, i] + train_x[:, i+1]*2 + train_x[:, i+28]*4 + train_x[:, i+29]*8


    train_x = numpy.array(new_train_x, dtype=int)

    test_labels = []
    with open('digitdata/testlabels', 'rb') as f:
        for i, line in enumerate(f):
            test_labels.append(int(line))
    test_labels = numpy.array(test_labels, dtype=int)

    test_x = numpy.zeros((test_labels.shape[0] * 28 * 28))
    with open('digitdata/testimages', 'rb') as f:
        for i, line in enumerate(f):
            for j, char in enumerate(line.strip('\n')):
                if '+' == char or '#' == char:
                    test_x[i * 28 + j] = 1

    test_x = test_x.reshape((test_labels.shape[0], 28 * 28))
    new_test_x = numpy.zeros((test_labels.shape[0], 27 * 27))
    for i in xrange(0, 27 * 27, 2):
        if i % 28 != 0 and i / 28 % 2 == 0:
            new_test_x[:, i] = test_x[:, i] + test_x[:, i+1]*2 + test_x[:, i+28]*4 + test_x[:, i+29]*8

    test_x = numpy.array(new_test_x, dtype=int)


    return train_x, train_labels, test_x, test_labels

class BayesClassifier(object):
    def __init__(self):
        self.bayesmatrix = None

    def fit(self, X, y):
        bayesmatrix = numpy.ones((10, 16, 27 * 27), dtype=numpy.float64)
        for k in xrange(10):
            for i in xrange(16):
                for j in xrange(X.shape[1]):
                    bayesmatrix[k, i, j] = numpy.sum(X[y==k, j]==i)
        numclass = numpy.zeros(10)
        for i in xrange(10):
            numclass[i] = numpy.sum(y==i) + 1
        bayesmatrix += 1.
        bayesmatrix /= numclass[:, numpy.newaxis, numpy.newaxis]
        self.bayesmatrix = bayesmatrix

    def predict(self, X):
        labels = []
        for i in xrange(X.shape[0]):<commit_msg>fix gf2 bug<commit_after>import numpy
from sklearn.metrics import confusion_matrix

def load_data():
    train_labels = []
    with open('digitdata/traininglabels', 'rb') as f:
        for i, line in enumerate(f):
            train_labels.append(int(line))
    train_labels = numpy.array(train_labels, dtype=int)

    train_x = numpy.zeros((train_labels.shape[0] * 28 * 28))
    with open('digitdata/trainingimages', 'rb') as f:
        for i, line in enumerate(f):
            for j, char in enumerate(line.strip('\n')):
                if '+' == char or '#' == char:
                    train_x[i * 28 + j] = 1

    train_x = train_x.reshape((train_labels.shape[0], 28 * 28))
    new_train_x = numpy.zeros((train_labels.shape[0], 14 * 14))
    cnt = 0
    for i in xrange(0, 27 * 27, 2):
        if i % 28 != 0 and i / 28 % 2 == 0:
            new_train_x[:, cnt] = train_x[:, i] + train_x[:, i+1]*2 + train_x[:, i+28]*4 + train_x[:, i+29]*8
            cnt += 1


    train_x = numpy.array(new_train_x, dtype=int)

    test_labels = []
    with open('digitdata/testlabels', 'rb') as f:
        for i, line in enumerate(f):
            test_labels.append(int(line))
    test_labels = numpy.array(test_labels, dtype=int)

    test_x = numpy.zeros((test_labels.shape[0] * 28 * 28))
    with open('digitdata/testimages', 'rb') as f:
        for i, line in enumerate(f):
            for j, char in enumerate(line.strip('\n')):
                if '+' == char or '#' == char:
                    test_x[i * 28 + j] = 1

    test_x = test_x.reshape((test_labels.shape[0], 28 * 28))
    new_test_x = numpy.zeros((test_labels.shape[0], 14 * 14))
    cnt = 0
    for i in xrange(0, 27 * 27, 2):
        if i % 28 != 0 and i / 28 % 2 == 0:
            new_test_x[:, cnt] = test_x[:, i] + test_x[:, i+1]*2 + test_x[:, i+28]*4 + test_x[:, i+29]*8
            cnt += 1

    test_x = numpy.array(new_test_x, dtype=int)


    return train_x, train_labels, test_x, test_labels

class BayesClassifier(object):
    def __init__(self):
        self.bayesmatrix = None

    def fit(self, X, y):
        bayesmatrix = numpy.ones((10, 16, 14 * 14), dtype=numpy.float64)
        for k in xrange(10):
            for i in xrange(16):
                for j in xrange(X.shape[1]):
                    bayesmatrix[k, i, j] = numpy.sum(X[y==k, j]==i)
        numclass = numpy.zeros(10)
        for i in xrange(10):
            numclass[i] = numpy.sum(y==i) + 1
        bayesmatrix += 1.
        bayesmatrix /= numclass[:, numpy.newaxis, numpy.newaxis]
        self.bayesmatrix = bayesmatrix

    def predict(self, X):
        labels = []
        for i in xrange(X.shape[0]):
{'fraction_non_alphanumeric': 0.1553769613471106, 'fraction_numerical': 0.03922694221201684, 'mean_word_length': 2.285355122564425, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1251122', 'n_tokens_mistral': 2033, 'n_tokens_neox': 1848, 'n_words': 467}