<commit_before>import glob
import os
from numpy import ndarray, fromfile, int16, uint16, prod
from matplotlib.pyplot import imread
from thunder.rdds.data import Data


class Images(Data):

    def __init__(self, rdd, dims=None):
        super(Images, self).__init__(rdd)
        self._dims = dims

    @property
    def dims(self):
        if self._dims is None:
            record = self.rdd.first()
            self._dims = record[1].shape
        return self._dims

    @staticmethod
    def _check_type(record):
        if not isinstance(record[0], tuple):
            raise Exception('Keys must be tuples')
        if not isinstance(record[1], ndarray):
            raise Exception('Values must be ndarrays')

    def toSeries(self):

        raise NotImplementedError

    def toBlocks(self):

        raise NotImplementedError

    def saveAsSeries(self):

        blocks = self.toBlocks()


class ImagesLoader(object):

    def __init__(self, sparkcontext, dims=None, valuetype=None, filerange=None):
        self.sc = sparkcontext
        self.dims = dims
        self.valuetype = valuetype
        self.filerange = filerange

    def fromStack(self, datafile, ext='stack'):

        def reader(file):
            f = open(file, 'rb')
            stack = fromfile(f, int16, prod(self.dims)).reshape(self.dims, order='F')
            f.close()
            return stack.astype(uint16)

        return Images(self.fromFile(datafile, reader, ext), dims=self.dims)

    def fromTif(self, datafile, ext='tif'):

        return Images(self.fromFile(datafile, imread, ext))

    def fromMultipageTif(self, datafile, ext='tif'):
        """
        Sets up a new Images object with data to be read from one or more multi-page tif files.

        The RDD underlying the returned Images will have key, value data as follows:

        key: tuple of int, int
            key[0] is index of original data file in lexicographic order
            key[1] is index of tif page within the original data file
        value: numpy ndarray

        This method attempts to explicitly import PIL. ImportError may be thrown if 'from PIL import Image' is
        unsuccessful. (PIL/pillow is not an explicit requirement for thunder.)
        """
        try:<commit_msg>move dims param to ImageLoader fromStack from init, remove unused valuetype<commit_after>import glob
import os
from numpy import ndarray, fromfile, int16, uint16, prod
from matplotlib.pyplot import imread
from thunder.rdds.data import Data


class Images(Data):

    def __init__(self, rdd, dims=None):
        super(Images, self).__init__(rdd)
        self._dims = dims

    @property
    def dims(self):
        if self._dims is None:
            record = self.rdd.first()
            self._dims = record[1].shape
        return self._dims

    @staticmethod
    def _check_type(record):
        if not isinstance(record[0], tuple):
            raise Exception('Keys must be tuples')
        if not isinstance(record[1], ndarray):
            raise Exception('Values must be ndarrays')

    def toSeries(self):

        raise NotImplementedError

    def toBlocks(self):

        raise NotImplementedError

    def saveAsSeries(self):

        blocks = self.toBlocks()


class ImagesLoader(object):

    def __init__(self, sparkcontext, filerange=None):
        self.sc = sparkcontext
        self.filerange = filerange

    def fromStack(self, datafile, dims, ext='stack'):

        def reader(filepath):
            f = open(filepath, 'rb')
            stack = fromfile(f, int16, prod(dims)).reshape(dims, order='F')
            f.close()
            return stack.astype(uint16)

        return Images(self.fromFile(datafile, reader, ext), dims=self.dims)

    def fromTif(self, datafile, ext='tif'):

        return Images(self.fromFile(datafile, imread, ext))

    def fromMultipageTif(self, datafile, ext='tif'):
        """
        Sets up a new Images object with data to be read from one or more multi-page tif files.

        The RDD underlying the returned Images will have key, value data as follows:

        key: tuple of int, int
            key[0] is index of original data file in lexicographic order
            key[1] is index of tif page within the original data file
        value: numpy ndarray

        This method attempts to explicitly import PIL. ImportError may be thrown if 'from PIL import Image' is
        unsuccessful. (PIL/pillow is not an explicit requirement for thunder.)
        """
        try:
{'fraction_non_alphanumeric': 0.0873634945397816, 'fraction_numerical': 0.005794517494985513, 'mean_word_length': 2.52276295133438, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1336861', 'n_tokens_mistral': 1310, 'n_tokens_neox': 1268, 'n_words': 430}