<commit_before>import numpy as np
import struct, gzip, os, sys
from collections import namedtuple

data = namedtuple('data',['X','Y'])
curdir = os.path.dirname(__file__)
datadir = os.path.join(curdir,'data')

def _read_header(filename, fmt, numBytes):
	f = gzip.GzipFile(filename,'rb')
	return struct.unpack(fmt,f.read(numBytes)),f.read()

def _labels_datatype():
	return np.dtype('>b')

def _images_datatype():
	return np.dtype('>B')

def _load_mnist(filenames):
	images_header,images_buffer = _read_header(filenames[0],'>IIII',16)
	labels_header,labels_buffer = _read_header(filenames[1],'>II',8)
	labels = np.fromstring(labels_buffer,dtype=_labels_datatype(),count=labels_header[1])
	images = np.fromstring(images_buffer,dtype=_images_datatype(),count=images_header[1]*images_header[2]*images_header[3]).reshape(images_header[1],images_header[2],images_header[3])
	return data(X=images,Y=labels)

def traindataset():
	return _load_mnist((os.path.abspath(os.path.join(datadir,'train-images-idx3-ubyte.gz')),
		os.path.abspath(os.path.join(datadir,'train-labels-idx1-ubyte.gz'))))


def testdataset():
	return _load_mnist((os.path.abspath(os.path.join(datadir,'t10k-images-idx3-ubyte.gz')),
		os.path.abspath(os.path.join(datadir,'t10k-labels-idx1-ubyte.gz'))))<commit_msg>reading into the buffer and closing the file.<commit_after>import numpy as np
import struct, gzip, os, sys
from collections import namedtuple

data = namedtuple('data',['X','Y'])
curdir = os.path.dirname(__file__)
datadir = os.path.join(curdir,'data')

def _read_header(filename, fmt, numBytes):
	f = gzip.GzipFile(filename,'rb')
	buff = f.read()
	f.close()
	return struct.unpack(fmt,f.read(numBytes)),

def _labels_datatype():
	return np.dtype('>b')

def _images_datatype():
	return np.dtype('>B')

def _load_mnist(filenames):
	images_header,images_buffer = _read_header(filenames[0],'>IIII',16)
	labels_header,labels_buffer = _read_header(filenames[1],'>II',8)
	labels = np.fromstring(labels_buffer,dtype=_labels_datatype(),count=labels_header[1])
	images = np.fromstring(images_buffer,dtype=_images_datatype(),count=images_header[1]*images_header[2]*images_header[3]).reshape(images_header[1],images_header[2],images_header[3])
	return data(X=images,Y=labels)

def traindataset():
	return _load_mnist((os.path.abspath(os.path.join(datadir,'train-images-idx3-ubyte.gz')),
		os.path.abspath(os.path.join(datadir,'train-labels-idx1-ubyte.gz'))))


def testdataset():
	return _load_mnist((os.path.abspath(os.path.join(datadir,'t10k-images-idx3-ubyte.gz')),
		os.path.abspath(os.path.join(datadir,'t10k-labels-idx1-ubyte.gz'))))
{'fraction_non_alphanumeric': 0.17115384615384616, 'fraction_numerical': 0.015384615384615385, 'mean_word_length': 8.289285714285715, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 11, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1928176', 'n_tokens_mistral': 1017, 'n_tokens_neox': 1019, 'n_words': 123}