<commit_before>import numpy as np

from mnist import MNIST

from classification_base import MNIST_PATH

def prep_binary_classes(y, digit=2):
    return np.array([int(x==digit) for x in y])

def mnist_training(shuffled=True):
    mndata = MNIST(MNIST_PATH)
    train_ims, train_labels = mndata.load_training()
    train_X = np.array(train_ims)
    train_y = np.array(train_labels)
    if shuffled:
        return shuffle(train_X, train_y)
    else:
        return train_X, train_y

def mnist_testing(shuffled = True):
    mndata = MNIST(MNIST_PATH)
    test_ims, test_labels = mndata.load_testing()
    test_X = np.array(test_ims)
    test_y = np.array(test_labels)
    if shuffled:
        return shuffle(test_X, test_y)
    else:
        return test_X, test_y

def mnist_training_binary(num, shuffled=True):
    X, y = mnist_training()
    if shuffled:
        X, y = shuffle(X, y)
    return X, np.array([int(yi==num) for yi in y])

def mnist_testing_binary(num):
    X, y = mnist_testing()
    return X, np.array([int(yi==num) for yi in y])

def shuffle(X, y):
    shuffler = np.arange(len(y))
    np.random.shuffle(shuffler)
    X = X[shuffler, :]
    y = y[shuffler]
    return X, y

def shuffle_train_and_test(X1, y1, X2, y2):
    print('Warning: shuffling train and test should not be used for work '
          'that is to be submitted.')
    np.random.seed(12345)
    all_X = np.concatenate((X1, X2))
    all_y = np.concatenate((y1, y2))
    X, y = shuffle(all_X, all_y)

    X1_shuffled = X[0:X1.shape[0], :]
    y1_shuffled = y[0:X1.shape[0]]
    X2_shuffled = X[X1.shape[0]:, :]
    y2_shuffled = y[X1.shape[0]:]
    return X1_shuffled, y1_shuffled, X2_shuffled, y2_shuffled

def binary_shuffled(num):
    X1, y1 = mnist_training_binary(num)
    X2, y2 = mnist_testing_binary(num)
    return shuffle_train_and_test(X1, y1, X2, y2)<commit_msg>simplify and transpose MNIST loading functions<commit_after>import numpy as np

from mnist import MNIST

MNIST_PATH = '../../data/python-mnist/data'

def mnist_training():
    """
    Returns MNIST training data with N columns and d rows
    """
    mndata = MNIST(MNIST_PATH)
    train_ims, train_labels = mndata.load_training()
    train_X = np.array(train_ims).T
    train_y = np.array(train_labels).T
    return train_X, train_y

def mnist_testing(shuffled = True):
    """
    Returns MNIST test data with N columns and d rows
    """
    mndata = MNIST(MNIST_PATH)
    test_ims, test_labels = mndata.load_testing()
    test_X = np.array(test_ims).T
    test_y = np.array(test_labels).T
    return test_X, test_y

def shuffle(X, y):
    shuffler = np.arange(len(y))
    np.random.shuffle(shuffler)
    X = X[:, shuffler]
    y = y[shuffler]
    return X, y
{'fraction_non_alphanumeric': 0.11680176860722181, 'fraction_numerical': 0.014738393515106854, 'mean_word_length': 3.3579454253611556, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3275091', 'n_tokens_mistral': 1093, 'n_tokens_neox': 1060, 'n_words': 241}