<commit_before>from __future__ import (absolute_import, division, print_function,
                        unicode_literals)
import six

import logging
import numpy as np
import os.path

import dask.array as da

from .handlers_base import HandlerBase
from pims import FramesSequence, Frame
from .readers.spe import PrincetonSPEFile

logger = logging.getLogger(__name__)


class IntegrityError(Exception):
    pass


class AreaDetectorSPEHandler(HandlerBase):
    specs = {'AD_SPE'} | HandlerBase.specs
    autoregister = True

    def __init__(self, fpath, template, filename,
                 frame_per_point=1):
        self._path = os.path.join(fpath, '')
        self._fpp = frame_per_point
        self._template = template
        self._filename = filename
        self._f_cache = dict()

    def __call__(self, point_number):
        if point_number not in self._f_cache:
            fname = self._template % (self._path,
                                      self._filename,
                                      point_number)
            spe_obj = PrincetonSPEFile(fname)
            self._f_cache[point_number] = spe_obj

        spe = self._f_cache[point_number]
        data = spe.getData()

        if data.shape[0] != self._fpp:
            raise IntegrityError("expected {} frames, found {} frames".format(
                                 self._fpp, data.shape[0]))
        return data.squeeze()

    def get_file_list(self, datum_kwarg_gen):
        return [self._template % (self._path,
                                  self._filename,
                                  d['point_number'])
                for d in datum_kwarg_gen]


class AreaDetectorTiffHandler(HandlerBase):
    specs = {'AD_TIFF'} | HandlerBase.specs
    autoregister = True

    def __init__(self, fpath, template, filename, frame_per_point=1):
        self._path = os.path.join(fpath, '')
        self._fpp = frame_per_point
        self._template = template
        self._filename = filename

    def _fnames_for_point(self, point_number):
        start = int(point_number * self._fpp)
        stop = int((point_number + 1) * self._fpp)
        for j in range(start, stop):
            yield self._template % (self._path, self._filename, j)

    def __call__(self, point_number):
        import tifffile
        ret = []
        for fn in self._fnames_for_point(point_number):
            with tifffile.TiffFile(fn) as tif:
                ret.append(tif.asarray())
        return np.array(ret).squeeze()

    def get_file_list(self, datum_kwargs):
        ret = []
        for d_kw in datum_kwargs:
            ret.extend(self._fnames_for_point(**d_kw))
        return ret


class DummyAreaDetectorHandler(HandlerBase):
    def __init__(self, fpath, frame_per_point=1, **kwargs):

        self._fpp = frame_per_point

    def __call__(self, **kwargs):
        out_stack = np.ones((self._fpp, 10, 10)) * np.nan
        # return stacked and squeezed results
        return out_stack.squeeze()


class ImageStack(FramesSequence):
    "One of these represents the data from an event: (num_images x w x h)"
    def __init__(self, dataset, start, stop):
        # `start` and `stop` are the limits of this cube
        # i indexes within the cube
        self._start = start
        self._stop = stop
        self._dataset = dataset

        # Old PIMS has no dtype, shape. Try adding it.
        try:
            self.dtype = self.pixel_type
        except AttributeError:
            pass
        try:
            self.shape = self.frame_shape
        except AttributeError:
            pass

    def get_frame(self, i):
        return Frame(self._dataset[self._start + i], frame_no=i)

    def __len__(self):
        return self._stop - self._start

    @property
    def pixel_type(self):
        return self._dataset.dtype

    @property
    def frame_shape(self):
        return self._dataset.shape[1:]


class HDF5DatasetSliceHandler(HandlerBase):
    """
    Handler for data stored in one Dataset of an HDF5 file.

    Parameters
    ----------
    filename : string
        path to HDF5 file
    key : string
        key of the single HDF5 Dataset used by this Handler
    frame_per_point : integer, optional
        number of frames to return as one datum, default 1
     swmr : bool, optional
        Open the hdf5 file in SWMR read mode. Only used when mode = 'r'.
    """
    def __init__(self, filename, key, frame_per_point=1, swmr=False):
        self._fpp = frame_per_point
        self._filename = filename
        self._key = key
        self._file = None
        self._dataset = None
        self._data_objects = dict()
        self._swmr = swmr

        logger.debug("filename = {}".format(filename))
        logger.debug("key = {}".format(key))
        logger.debug("frame_per_point = {}".format(frame_per_point))
        logger.debug("swmr = {}".format(swmr))

        self.open()

    def get_file_list(self, datum_kwarg_gen):
        return [self._filename]

    def __call__(self, point_number):
        # Don't read out the dataset until it is requested for the first time.<commit_msg>removed stray space<commit_after>from __future__ import (absolute_import, division, print_function,
                        unicode_literals)
import logging
import numpy as np
import os.path

import dask.array as da

from .handlers_base import HandlerBase
from pims import FramesSequence, Frame
from .readers.spe import PrincetonSPEFile

logger = logging.getLogger(__name__)


class IntegrityError(Exception):
    pass


class AreaDetectorSPEHandler(HandlerBase):
    specs = {'AD_SPE'} | HandlerBase.specs
    autoregister = True

    def __init__(self, fpath, template, filename,
                 frame_per_point=1):
        self._path = os.path.join(fpath, '')
        self._fpp = frame_per_point
        self._template = template
        self._filename = filename
        self._f_cache = dict()

    def __call__(self, point_number):
        if point_number not in self._f_cache:
            fname = self._template % (self._path,
                                      self._filename,
                                      point_number)
            spe_obj = PrincetonSPEFile(fname)
            self._f_cache[point_number] = spe_obj

        spe = self._f_cache[point_number]
        data = spe.getData()

        if data.shape[0] != self._fpp:
            raise IntegrityError("expected {} frames, found {} frames".format(
                                 self._fpp, data.shape[0]))
        return data.squeeze()

    def get_file_list(self, datum_kwarg_gen):
        return [self._template % (self._path,
                                  self._filename,
                                  d['point_number'])
                for d in datum_kwarg_gen]


class AreaDetectorTiffHandler(HandlerBase):
    specs = {'AD_TIFF'} | HandlerBase.specs
    autoregister = True

    def __init__(self, fpath, template, filename, frame_per_point=1):
        self._path = os.path.join(fpath, '')
        self._fpp = frame_per_point
        self._template = template
        self._filename = filename

    def _fnames_for_point(self, point_number):
        start = int(point_number * self._fpp)
        stop = int((point_number + 1) * self._fpp)
        for j in range(start, stop):
            yield self._template % (self._path, self._filename, j)

    def __call__(self, point_number):
        import tifffile
        ret = []
        for fn in self._fnames_for_point(point_number):
            with tifffile.TiffFile(fn) as tif:
                ret.append(tif.asarray())
        return np.array(ret).squeeze()

    def get_file_list(self, datum_kwargs):
        ret = []
        for d_kw in datum_kwargs:
            ret.extend(self._fnames_for_point(**d_kw))
        return ret


class DummyAreaDetectorHandler(HandlerBase):
    def __init__(self, fpath, frame_per_point=1, **kwargs):

        self._fpp = frame_per_point

    def __call__(self, **kwargs):
        out_stack = np.ones((self._fpp, 10, 10)) * np.nan
        # return stacked and squeezed results
        return out_stack.squeeze()


class ImageStack(FramesSequence):
    "One of these represents the data from an event: (num_images x w x h)"
    def __init__(self, dataset, start, stop):
        # `start` and `stop` are the limits of this cube
        # i indexes within the cube
        self._start = start
        self._stop = stop
        self._dataset = dataset

        # Old PIMS has no dtype, shape. Try adding it.
        try:
            self.dtype = self.pixel_type
        except AttributeError:
            pass
        try:
            self.shape = self.frame_shape
        except AttributeError:
            pass

    def get_frame(self, i):
        return Frame(self._dataset[self._start + i], frame_no=i)

    def __len__(self):
        return self._stop - self._start

    @property
    def pixel_type(self):
        return self._dataset.dtype

    @property
    def frame_shape(self):
        return self._dataset.shape[1:]


class HDF5DatasetSliceHandler(HandlerBase):
    """
    Handler for data stored in one Dataset of an HDF5 file.

    Parameters
    ----------
    filename : string
        path to HDF5 file
    key : string
        key of the single HDF5 Dataset used by this Handler
    frame_per_point : integer, optional
        number of frames to return as one datum, default 1
    swmr : bool, optional
        Open the hdf5 file in SWMR read mode. Only used when mode = 'r'.
    """
    def __init__(self, filename, key, frame_per_point=1, swmr=False):
        self._fpp = frame_per_point
        self._filename = filename
        self._key = key
        self._file = None
        self._dataset = None
        self._data_objects = dict()
        self._swmr = swmr

        logger.debug("filename = {}".format(filename))
        logger.debug("key = {}".format(key))
        logger.debug("frame_per_point = {}".format(frame_per_point))
        logger.debug("swmr = {}".format(swmr))

        self.open()

    def get_file_list(self, datum_kwarg_gen):
        return [self._filename]

    def __call__(self, point_number):
        # Don't read out the dataset until it is requested for the first time.
{'fraction_non_alphanumeric': 0.09164949465214405, 'fraction_numerical': 0.003532528701795702, 'mean_word_length': 2.158351409978308, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 10, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7032173', 'n_tokens_mistral': 3033, 'n_tokens_neox': 2928, 'n_words': 789}