<commit_before>#!/usr/bin/env python
# -*- coding: UTF-8 -*-
#
# Author : @Jason_wbw

"""
Converter MNIST data to np.array

MNIST files are available at http://yann.lecun.com/exdb/mnist/
Before using this program you first need to download files:
train-images.idx3-ubyte.gz : train image data
t10k-images.idx3-ubyte.gz  : test image data
train-labels.idx1-ubyte.gz : train labels data
t10k-labels.idx1-ubyte.gz  : test labels data
and gunzip them.
"""

import numpy as np
import struct
import matplotlib.pyplot as plt

class Converter(object):


	def __init__(self):
		#self.test_read('t10k-images.idx3-ubyte')
		train_images = self.read_images('train-images.idx3-ubyte')
		test_images = self.read_images('t10k-images.idx3-ubyte')
		self.train_labels = self.read_labels('train-labels.idx1-ubyte')
		self.test_labels = self.read_labels('t10k-labels.idx1-ubyte')
		self.train_images, self.test_images = self.simple_scale(train_images, test_images)
		self.total_train_data = len(self.train_labels)
		self.dimensionality = 28 * 28


	def simple_scale(self, data0, data1):
		return data0 / 255, data1 / 255

	def test_read(self, filename):
		images = []

		binfile = open(filename , 'rb')
		buf = binfile.read()

		index = 0
		magic, numImages, numRows, numColumns = struct.unpack_from('>IIII', buf, index) #read 4unsinged int32
		index += struct.calcsize('>IIII')

		im = struct.unpack_from('>%dB' % (28*28), buf, index) #read 784unsigned byte
		index += struct.calcsize('>%dB' % (28*28))

		im = np.array(im)
		im = im.reshape(28, 28) 
		fig = plt.figure()
		plotwindow = fig.add_subplot(111)
		plt.imshow(im , cmap='gray')
		plt.show()


	def read_images(self, filename):
		images = []

		binfile = open(filename , 'rb')
		buf = binfile.read()

		index = 0
		magic, numImages, numRows, numColumns = struct.unpack_from('>IIII', buf, index) #read 4 unsinged int32
		index += struct.calcsize('>IIII')

		for i in xrange(numImages):
			im = struct.unpack_from('>%dB' % (28*28), buf, index) #read 784unsigned byte
			index += struct.calcsize('>%dB' % (28*28))

			im = np.array(im)
			images.append(im)

		return np.array(images)


	def read_labels(self, filename):
		labels = []

		binfile = open(filename , 'rb')
		buf = binfile.read()

		index = 0
		magic, numLabels = struct.unpack_from('>II', buf, index) #read 2unsinged int32
		index += struct.calcsize('>II')

		for i in xrange(numLabels):
			im = struct.unpack_from('>B' ,buf, index) #read 1 unsigned byte
			index += struct.calcsize('>B')

			im = np.array(im)
			labels.append(im)

		return np.array(labels)<commit_msg>fix normalization bug<commit_after>#!/usr/bin/env python
# -*- coding: UTF-8 -*-
#
# Author : @Jason_wbw

"""
Converter MNIST data to np.array

MNIST files are available at http://yann.lecun.com/exdb/mnist/
Before using this program you first need to download files:
train-images.idx3-ubyte.gz : train image data
t10k-images.idx3-ubyte.gz  : test image data
train-labels.idx1-ubyte.gz : train labels data
t10k-labels.idx1-ubyte.gz  : test labels data
and gunzip them.
"""

import numpy as np
import struct
import matplotlib.pyplot as plt

class Converter(object):


	def __init__(self):
		#self.test_read('t10k-images.idx3-ubyte')
		train_images = self.read_images('train-images.idx3-ubyte')
		test_images = self.read_images('t10k-images.idx3-ubyte')
		self.train_labels = self.read_labels('train-labels.idx1-ubyte')
		self.test_labels = self.read_labels('t10k-labels.idx1-ubyte')
		self.train_images, self.test_images = self.simple_scale(train_images, test_images)
		self.total_train_data = len(self.train_labels)
		self.dimensionality = 28 * 28

	def simple_scale(self, data0, data1):
		return data0 / 255., data1 / 255.

	def test_read(self, filename):
		images = []

		binfile = open(filename , 'rb')
		buf = binfile.read()

		index = 0
		magic, numImages, numRows, numColumns = struct.unpack_from('>IIII', buf, index) #read 4unsinged int32
		index += struct.calcsize('>IIII')

		im = struct.unpack_from('>%dB' % (28*28), buf, index) #read 784unsigned byte
		index += struct.calcsize('>%dB' % (28*28))

		im = np.array(im)
		im = im.reshape(28, 28) 
		fig = plt.figure()
		plotwindow = fig.add_subplot(111)
		plt.imshow(im , cmap='gray')
		plt.show()

	def read_images(self, filename):
		images = []

		binfile = open(filename , 'rb')
		buf = binfile.read()

		index = 0
		magic, numImages, numRows, numColumns = struct.unpack_from('>IIII', buf, index) #read 4 unsinged int32
		index += struct.calcsize('>IIII')

		for i in xrange(numImages):
			im = struct.unpack_from('>%dB' % (28*28), buf, index) #read 784unsigned byte
			index += struct.calcsize('>%dB' % (28*28))

			im = np.array(im)
			images.append(im)

		return np.array(images)

	def read_labels(self, filename):
		labels = []

		binfile = open(filename , 'rb')
		buf = binfile.read()

		index = 0
		magic, numLabels = struct.unpack_from('>II', buf, index) #read 2unsinged int32
		index += struct.calcsize('>II')

		for i in xrange(numLabels):
			im = struct.unpack_from('>B' ,buf, index) #read 1 unsigned byte
			index += struct.calcsize('>B')

			im = np.array(im)
			labels.append(im)

		return np.array(labels)
{'fraction_non_alphanumeric': 0.1471558920597942, 'fraction_numerical': 0.029508833236264805, 'mean_word_length': 4.070866141732283, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 27, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '95517', 'n_tokens_mistral': 2162, 'n_tokens_neox': 1916, 'n_words': 435}