<commit_before># took file format from from http://abel.ee.ucla.edu/cvxopt/_downloads/mnist.py
#to whom credit is due.
import os
import struct
import gzip
import numpy
import operator

def read_mnist(digits, dataset="training"):
    """
    Read the mnist data set. 
    Data set obtained from http://yann.lecun.com/exdb/mnist/ .
    
    Parameters: 
    digits - list of digits to read
    dataset - either "training" or "testing"
    """
    assert dataset in ['testing','training']
    filename_prefix = {'testing':'t10k', 'training':'train'}
    path = os.path.dirname(__file__)
    filename_images = os.path.join(path, '%s-images-idx3-ubyte.gz'%filename_prefix[dataset])
    filename_labels = os.path.join(path, '%s-labels-idx1-ubyte.gz'%filename_prefix[dataset])

    labelfile = gzip.open(filename_labels, 'rb')
    magic_nr, size = struct.unpack(">II", labelfile.read(8))
    labelstring = labelfile.read()
    labelfile.close()
    labels = numpy.fromstring(labelstring, dtype=numpy.int8)

    imagefile = gzip.open(filename_images, 'rb')
    magic_nr, size, rows, cols = struct.unpack(">IIII", imagefile.read(16))
    imagestring = imagefile.read()
    imagefile.close()
    images = numpy.fromstring(imagestring, dtype=numpy.uint8)
    images = images.reshape((size, len(images)/size))

    index_list = [labels == digit for digit in digits ]
    indices = reduce(operator.or_, index_list)
    retimg = images[indices,:]
    retlab = labels[indices]

    return retimg, retlab

def get_training_data(digits, num_samples):
    """
    Read num_samples training instances of the digits from the MNIST database.

    Returns 2-tuple (vec, lab) with vec a num_samples X 784 array of pixel grey values,
    and lab an num_samples length array of labels.
    
    Parameters:
    digits - list of digits to read (list of int)
    num_samples - number of samples (int)
    """
    img,lab = read_mnist(digits, dataset='training')
    img = img[:num_samples,:]
    lab = lab[:num_samples]
    return img, lab
    
def get_testing_data(digits, num_samples):
    """
    Read num_samples testing instances of the digits from the MNIST database.

    Returns 2-tuple (vec, lab) with vec a num_samples X 784 array of pixel grey values,
    and lab an num_samples length array of labels.
    
    Parameters:
    digits - list of digits to read (list of int)
    num_samples - number of samples (int)
    """
    img,data = read_mnist(digits, dataset='testing')
    img = img[:num_samples,:]
    lab = lab[:num_samples]<commit_msg>fixed potential glitch in get_testing_data<commit_after># took file format from from http://abel.ee.ucla.edu/cvxopt/_downloads/mnist.py
#to whom credit is due.
import os
import struct
import gzip
import numpy
import operator

def read_mnist(digits, dataset="training"):
    """
    Read the mnist data set. 
    Data set obtained from http://yann.lecun.com/exdb/mnist/ .
    
    Parameters: 
    digits - list of digits to read
    dataset - either "training" or "testing"
    """
    assert dataset in ['testing','training']
    filename_prefix = {'testing':'t10k', 'training':'train'}
    path = os.path.dirname(__file__)
    filename_images = os.path.join(path, '%s-images-idx3-ubyte.gz'%filename_prefix[dataset])
    filename_labels = os.path.join(path, '%s-labels-idx1-ubyte.gz'%filename_prefix[dataset])

    labelfile = gzip.open(filename_labels, 'rb')
    magic_nr, size = struct.unpack(">II", labelfile.read(8))
    labelstring = labelfile.read()
    labelfile.close()
    labels = numpy.fromstring(labelstring, dtype=numpy.int8)

    imagefile = gzip.open(filename_images, 'rb')
    magic_nr, size, rows, cols = struct.unpack(">IIII", imagefile.read(16))
    imagestring = imagefile.read()
    imagefile.close()
    images = numpy.fromstring(imagestring, dtype=numpy.uint8)
    images = images.reshape((size, len(images)/size))

    index_list = [labels == digit for digit in digits ]
    indices = reduce(operator.or_, index_list)
    retimg = images[indices,:]
    retlab = labels[indices]

    return retimg, retlab

def get_training_data(digits, num_samples):
    """
    Read num_samples training instances of the digits from the MNIST database.

    Returns 2-tuple (vec, lab) with vec a num_samples X 784 array of pixel grey values,
    and lab an num_samples length array of labels.
    
    Parameters:
    digits - list of digits to read (list of int)
    num_samples - number of samples (int)
    """
    img,lab = read_mnist(digits, dataset='training')
    img = img[:num_samples,:]
    lab = lab[:num_samples]
    return img, lab
    
def get_testing_data(digits, num_samples):
    """
    Read num_samples testing instances of the digits from the MNIST database.

    Returns 2-tuple (vec, lab) with vec a num_samples X 784 array of pixel grey values,
    and lab an num_samples length array of labels.
    
    Parameters:
    digits - list of digits to read (list of int)
    num_samples - number of samples (int)
    """
    img,lab = read_mnist(digits, dataset='testing')
    img = img[:num_samples,:]
    lab = lab[:num_samples]
{'fraction_non_alphanumeric': 0.11242138364779874, 'fraction_numerical': 0.0066823899371069185, 'mean_word_length': 3.491615180935569, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2196189', 'n_tokens_mistral': 1667, 'n_tokens_neox': 1608, 'n_words': 465}