<commit_before>import numpy as np
from scipy import misc
import glob
from tensorflow.contrib.learn.python.learn.datasets import base


class DataSet(object):

	def __init__(self, images, labels):
		self._images = images
		self._labels = labels
		self._num_examples = images.shape[0]
		self._epochs_completed = 0
		self._index_in_epoch = 0

	@property
	def images(self):
		return self._images

	@property
	def labels(self):
		return self._labels

	@property
	def num_examples(self):
		return self._num_examples

	@property
	def epochs_completed(self):
		return self._epochs_completed

	def next_batch(self, batch_size):
		"""Return the next `batch_size` examples from this data set."""
		start = self._index_in_epoch
		self._index_in_epoch += batch_size
		if self._index_in_epoch > self._num_examples:
			# Finished epoch
			self._epochs_completed += 1
			# Shuffle the data
			perm = np.arange(self._num_examples)
			np.random.shuffle(perm)
			self._images = self._images[perm]
			self._labels = self._labels[perm]
			# Start next epoch
			start = 0
			self._index_in_epoch = batch_size
			assert batch_size <= self._num_examples
		end = self._index_in_epoch
		return self._images[start:end], self._labels[start:end]


def read_data_set():

	image_paths = glob.glob("./IMG/*.png")

	train_inputs = []
	train_targets = []

	# load data into train_inputs/targets
	for itr in range(int(len(image_paths)/3)):
		i = itr*3
		
		before_target = np.array(misc.imread(image_paths[i]))
		target = np.array(misc.imread(image_paths[i+1]))
		after_target = np.array(misc.imread(image_paths[i+2]))
		
		x = np.concatenate((before_target,after_target),axis = 2)

		train_inputs.append(x)
		train_targets.append(target)
		
	train_inputs = np.array(train_inputs)
	train_targets = np.array(train_targets)
	print(train_inputs.shape)


	## split into train, test, validation
	dataset_size = len(train_inputs)
	test_size = int(0.15*dataset_size)
	validation_size = test_size

	# shuffle data<commit_msg>fixed data loading so we get more data<commit_after>import numpy as np
from scipy import misc
import glob
from tensorflow.contrib.learn.python.learn.datasets import base


class DataSet(object):

	def __init__(self, images, labels):
		self._images = images
		self._labels = labels
		self._num_examples = images.shape[0]
		self._epochs_completed = 0
		self._index_in_epoch = 0

	@property
	def images(self):
		return self._images

	@property
	def labels(self):
		return self._labels

	@property
	def num_examples(self):
		return self._num_examples

	@property
	def epochs_completed(self):
		return self._epochs_completed

	def next_batch(self, batch_size):
		"""Return the next `batch_size` examples from this data set."""
		start = self._index_in_epoch
		self._index_in_epoch += batch_size
		if self._index_in_epoch > self._num_examples:
			# Finished epoch
			self._epochs_completed += 1
			# Shuffle the data
			perm = np.arange(self._num_examples)
			np.random.shuffle(perm)
			self._images = self._images[perm]
			self._labels = self._labels[perm]
			# Start next epoch
			start = 0
			self._index_in_epoch = batch_size
			assert batch_size <= self._num_examples
		end = self._index_in_epoch
		return self._images[start:end], self._labels[start:end]


def read_data_set():

	image_paths = glob.glob("./IMG/*.png")

	train_inputs = []
	train_targets = []

	# load data into train_inputs/targets
	for i in range(len(image_paths)-2):
		
		before_target = np.array(misc.imread(image_paths[i]))
		target = np.array(misc.imread(image_paths[i+1]))
		after_target = np.array(misc.imread(image_paths[i+2]))
		
		x = np.concatenate((before_target,after_target),axis = 2)

		train_inputs.append(x)
		train_targets.append(target)
		
	train_inputs = np.array(train_inputs)
	train_targets = np.array(train_targets)
	print(train_inputs.shape)


	## split into train, test, validation
	dataset_size = len(train_inputs)
	test_size = int(0.15*dataset_size)
	validation_size = test_size

	# shuffle data
{'fraction_non_alphanumeric': 0.10680100755667506, 'fraction_numerical': 0.006297229219143577, 'mean_word_length': 4.492392807745505, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5781059', 'n_tokens_mistral': 1524, 'n_tokens_neox': 1354, 'n_words': 290}