<commit_before>"""
Dataset class that wraps the dataset class found in 
image_directory_dataset to support models with branched
input layers; allowing different versions of images as
input to those layers.

Based on dev work in the Interactive Pylearn2 notebook.
"""
__authors__ = "<NAME>"
__copyright__ = "Copyright 2015 - University of Edinburgh"
__credits__ = ["<NAME>"]
__license__ = "MIT"
__maintainer__ = "<NAME>"
__email__ = "<EMAIL>"


import numpy as np

import neukrill_net.image_directory_dataset

class ParallelIterator(neukrill_net.image_directory_dataset.FlyIterator):
    """
    A simple version of FlyIterator that is able to deal with multiple
    images being returned by the processing function.
    """
    def next(self):
        # return one batch
        if len(self.indices) >= self.batch_size:
            batch_indices = [self.indices.pop(0) for i in range(self.batch_size)]
            # preallocate array
            if len(self.final_shape) == 2: 
                Xbatch1 = np.zeros([self.batch_size]+list(self.final_shape)+[1])
                Xbatch2 = np.zeros([self.batch_size]+list(self.final_shape)+[1])
            elif len(self.final_shape) == 3:
                Xbatch1 = np.zeros([self.batch_size]+list(self.final_shape))
                Xbatch2 = np.zeros([self.batch_size]+list(self.final_shape))
            # iterate over indices, applying the dataset's processing function
            for i,j in enumerate(batch_indices):
                import pdb
                pdb.set_trace()
                Xbatch1[i],Xbatch2[i] = [image.reshape(Xbatch1.shape[1:]) for 
                        image in self.dataset.fn(self.dataset.X[j])]
            # get the batch for y as well
            ybatch = self.dataset.y[batch_indices,:].astype(np.float32)
            Xbatch1 = Xbatch1.astype(np.float32)
            Xbatch2 = Xbatch2.astype(np.float32)
            return Xbatch1,Xbatch2,ybatch
        else:
            raise StopIteration<commit_msg>pdb no longer required<commit_after>"""
Dataset class that wraps the dataset class found in 
image_directory_dataset to support models with branched
input layers; allowing different versions of images as
input to those layers.

Based on dev work in the Interactive Pylearn2 notebook.
"""
__authors__ = "<NAME>"
__copyright__ = "Copyright 2015 - University of Edinburgh"
__credits__ = ["<NAME>"]
__license__ = "MIT"
__maintainer__ = "<NAME>"
__email__ = "<EMAIL>"


import numpy as np

import neukrill_net.image_directory_dataset

class ParallelIterator(neukrill_net.image_directory_dataset.FlyIterator):
    """
    A simple version of FlyIterator that is able to deal with multiple
    images being returned by the processing function.
    """
    def next(self):
        # return one batch
        if len(self.indices) >= self.batch_size:
            batch_indices = [self.indices.pop(0) for i in range(self.batch_size)]
            # preallocate array
            if len(self.final_shape) == 2: 
                Xbatch1 = np.zeros([self.batch_size]+list(self.final_shape)+[1])
                Xbatch2 = np.zeros([self.batch_size]+list(self.final_shape)+[1])
            elif len(self.final_shape) == 3:
                Xbatch1 = np.zeros([self.batch_size]+list(self.final_shape))
                Xbatch2 = np.zeros([self.batch_size]+list(self.final_shape))
            # iterate over indices, applying the dataset's processing function
            for i,j in enumerate(batch_indices):
                Xbatch1[i],Xbatch2[i] = [image.reshape(Xbatch1.shape[1:]) for 
                        image in self.dataset.fn(self.dataset.X[j])]
            # get the batch for y as well
            ybatch = self.dataset.y[batch_indices,:].astype(np.float32)
            Xbatch1 = Xbatch1.astype(np.float32)
            Xbatch2 = Xbatch2.astype(np.float32)
            return Xbatch1,Xbatch2,ybatch
        else:
            raise StopIteration
{'fraction_non_alphanumeric': 0.09923469387755102, 'fraction_numerical': 0.015306122448979591, 'mean_word_length': 2.6920903954802258, 'pattern_counts': {'":': 0, '<': 11, '<?xml version=': 0, '>': 13, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1534638', 'n_tokens_mistral': 1203, 'n_tokens_neox': 1140, 'n_words': 297}