<commit_before>#!/usr/bin/env python

import sys
import numpy
import math
import logging

from fuel.datasets import DogsVsCats
from fuel.schemes import ShuffledScheme
from fuel.server import start_server
from fuel.streams import DataStream
from fuel.transformers import ScaleAndShift, ForceFloatX
from fuel.transformers.image import (
    RandomFixedSizeCrop,
    SourcewiseTransformer, ExpectsAxisLabels)

from PIL import Image

class ForceMinimumDimension(SourcewiseTransformer, ExpectsAxisLabels):
    def __init__(self, data_stream, min_dim, resample='nearest',
                 **kwargs):
        self.min_dim = min_dim
        try:
            self.resample = getattr(Image, resample.upper())
        except AttributeError:
            raise ValueError("unknown resampling filter '{}'".format(resample))
        kwargs.setdefault('produces_examples', data_stream.produces_examples)
        kwargs.setdefault('axis_labels', data_stream.axis_labels)
        super(ForceMinimumDimension, self).__init__(data_stream, **kwargs)

    def transform_source_batch(self, batch, source_name):
        self.verify_axis_labels(('batch', 'channel', 'height', 'width'),
                                self.data_stream.axis_labels[source_name],
                                source_name)
        return [self._example_transform(im, source_name) for im in batch]

    def transform_source_example(self, example, source_name):
        self.verify_axis_labels(('channel', 'height', 'width'),
                                self.data_stream.axis_labels[source_name],
                                source_name)
        return self._example_transform(example, source_name)

    def _example_transform(self, example, _):
        if example.ndim > 3 or example.ndim < 2:
            raise NotImplementedError
        original_min_dim = min(example.shape[-2:])
        multiplier = self.min_dim / float(original_min_dim)
        dt = example.dtype
        # If we're dealing with a colour image, swap around the axes
        # to be in the format that PIL needs.
        if example.ndim == 3:
            im = example.transpose(1, 2, 0)
        else:
            im = example
        im = Image.fromarray(im)
        width, height = im.size
        width = int(math.ceil(width * multiplier))
        height = int(math.ceil(height * multiplier))
        im = numpy.array(im.resize((width, height))).astype(dt)
        # If necessary, undo the axis swap from earlier.
        if im.ndim == 3:
            example = im.transpose(2, 0, 1)
        else:
            example = im
        return example


def add_transformers(stream, random_crop=False):
    stream = ForceMinimumDimension(stream, 128,
                                   which_sources=['image_features'])
    if random_crop:
        stream = RandomFixedSizeCrop(stream, (128, 128),
                                    which_sources=['image_features'])
    stream = ScaleAndShift(stream, 1 / 255.0, 0,
                           which_sources=['image_features'])
    stream = ForceFloatX(stream)
    return stream

if __name__ == '__main__':
    logging.basicConfig(level=logging.DEBUG)
    train = DogsVsCats(("train",), subset=slice(None, int(sys.argv[1]), None))
    train_str =  DataStream(
        train, iteration_scheme=ShuffledScheme(train.num_examples, int(sys.argv[2])))
    train_str = add_transformers(train_str, random_crop=True)
    start_server(train_str, port=int(sys.argv[3]))<commit_msg>Assume the dimesions are good<commit_after>#!/usr/bin/env python

import sys
import numpy
import math
import logging

from fuel.datasets import DogsVsCats
from fuel.schemes import ShuffledScheme
from fuel.server import start_server
from fuel.streams import DataStream
from fuel.transformers import ScaleAndShift, ForceFloatX
from fuel.transformers.image import (
    RandomFixedSizeCrop,
    SourcewiseTransformer, ExpectsAxisLabels)

from PIL import Image

class ForceMinimumDimension(SourcewiseTransformer, ExpectsAxisLabels):
    def __init__(self, data_stream, min_dim, resample='nearest',
                 **kwargs):
        self.min_dim = min_dim
        try:
            self.resample = getattr(Image, resample.upper())
        except AttributeError:
            raise ValueError("unknown resampling filter '{}'".format(resample))
        kwargs.setdefault('produces_examples', data_stream.produces_examples)
        kwargs.setdefault('axis_labels', data_stream.axis_labels)
        super(ForceMinimumDimension, self).__init__(data_stream, **kwargs)

    def transform_source_batch(self, batch, source_name):
        self.verify_axis_labels(('batch', 'channel', 'height', 'width'),
                                self.data_stream.axis_labels[source_name],
                                source_name)
        return [self._example_transform(im, source_name) for im in batch]

    def transform_source_example(self, example, source_name):
        self.verify_axis_labels(('channel', 'height', 'width'),
                                self.data_stream.axis_labels[source_name],
                                source_name)
        return self._example_transform(example, source_name)

    def _example_transform(self, example, _):
        if example.ndim > 3 or example.ndim < 2:
            raise NotImplementedError
        original_min_dim = min(example.shape[-2:])
        multiplier = self.min_dim / float(original_min_dim)
        dt = example.dtype
        # If we're dealing with a colour image, swap around the axes
        # to be in the format that PIL needs.
        if example.ndim == 3:
            im = example.transpose(1, 2, 0)
        else:
            im = example
        im = Image.fromarray(im)
        width, height = im.size
        width = int(math.ceil(width * multiplier))
        height = int(math.ceil(height * multiplier))
        im = numpy.array(im.resize((width, height))).astype(dt)
        # If necessary, undo the axis swap from earlier.
        if im.ndim == 3:
            example = im.transpose(2, 0, 1)
        else:
            example = im
        return example


def add_transformers(stream, random_crop=False):
    # Now the dataset has images with good minimum size
    # stream = ForceMinimumDimension(stream, 128,
    #                                which_sources=['image_features'])
    if random_crop:
        stream = RandomFixedSizeCrop(stream, (128, 128),
                                    which_sources=['image_features'])
    stream = ScaleAndShift(stream, 1 / 255.0, 0,
                           which_sources=['image_features'])
    stream = ForceFloatX(stream)
    return stream

if __name__ == '__main__':
    logging.basicConfig(level=logging.DEBUG)
    train = DogsVsCats(("train",), subset=slice(None, int(sys.argv[1]), None))
    train_str =  DataStream(
        train, iteration_scheme=ShuffledScheme(train.num_examples, int(sys.argv[2])))
    train_str = add_transformers(train_str, random_crop=True)
    start_server(train_str, port=int(sys.argv[3]))
{'fraction_non_alphanumeric': 0.09673258813413585, 'fraction_numerical': 0.00831183720263686, 'mean_word_length': 2.648196549921589, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2908043', 'n_tokens_mistral': 2041, 'n_tokens_neox': 1951, 'n_words': 458}