<commit_before>__author__ = "<NAME> <<EMAIL>>"
__date__ = "$Jun 09, 2014 8:51:33AM$"


# This program is free software; you can redistribute it and/or
# modify it under the terms of the GNU General Public License
# as published by the Free Software Foundation; either version 2
# of the License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software Foundation,
# Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
#
# Copyright 2011-2014, the ilastik developers

"""High-level API.

"""


import debugging_tools

logger = debugging_tools.logging.getLogger(__name__)

import os
import collections
import itertools
import threading

import h5py
import numpy

from PyQt4.QtGui import QApplication
from PyQt4.QtCore import pyqtSignal
from PyQt4.QtCore import QObject

from volumina.multimethods import multimethod
from volumina.pixelpipeline.datasources import SourceABC, RequestABC
from volumina.pixelpipeline.datasources import is_pure_slicing
from volumina.layer import GrayscaleLayer, RGBALayer, ColortableLayer, ClickableColortableLayer, AlphaModulatedLayer
from volumina.viewer import Viewer

import HDF5_searchers
import expanded_numpy
import additional_generators



class HDF5DatasetNotFoundException( Exception ):
    """
        An exception raised when a dataset is not found in an HDF5 file.
    """
    pass

class HDF5DataSource( QObject ):
    """
        Creates a source that reads from an HDF5 dataset and shapes it in a way that Volumina can use.

        Attributes:
              file_handle(h5py.File or str):           A handle for reading the HDF5 file or the external file path.
              file_path(str):                          External path to the file
              dataset_path(str):                       Internal path to the dataset
              full_path(str):                          Both external and internal paths combined as one path
              dataset_shape(tuple of ints):            A tuple representing the shape of the dataset in each dimension
              dataset_dtype(numpy.dtype or type):      The type of the underlying dataset.
              axis_order(tuple of ints):               A tuple representing how to reshape the array before returning a request

    """

    #TODO: Reshaping should probably be some sort of lazyflow operator and thus removed from this directly.

    isDirty = pyqtSignal( object )
    numberOfChannelsChanged = pyqtSignal(int) # Never emitted

    @debugging_tools.log_call(logger)
    def __init__( self, file_handle, internal_path, shape = None, dtype = None):
        """
            Constructs an HDF5DataSource using a given file and path to the dataset. Optionally, the shape and dtype
            can be specified.

            Args:
                file_handle(h5py.File or str):          A file handle for the HDF5 file..
                internal_path(str):                     path to the dataset inside of the HDF5 file.
                shape(tuple of ints):                   shape of underlying dataset if not specified defaults to that of the dataset.
                dtype(numpy.dtype or type):             type of underlying dataset if not specified defaults to that of the dataset.
        """
        #TODO: Get rid of shape and dtype as arguments.

        super(HDF5DataSource, self).__init__()

        self.file_handle = None

        self.file_path = ""
        self.dataset_path = ""

        self.full_path = ""

        # Constructed standard shape and dtype if provided
        self.dataset_shape = tuple(shape) if shape is not None else None
        self.dataset_dtype = numpy.dtype(dtype).type if dtype is not None else None

        self.axis_order = [-1, -1, -1, -1, -1]

        # If it is a filename, get the file handle.
        if isinstance(file_handle, str):
            file_handle.rstrip("/")
            file_handle = h5py.File(file_handle)

        self.file_handle = file_handle

        self.file_path = self.file_handle.filename
        self.dataset_path = "/" + internal_path.strip("/")

        self.full_path = self.file_path + self.dataset_path

        # Check to see if the dataset exists in the file. Otherwise throw an Exception for it.
        if self.dataset_path not in self.file_handle:
            raise(HDF5DatasetNotFoundException("Could not find the path \"" + self.dataset_path + "\" in filename " + "\"" + self.file_path + "\"."))

        # Fill in the shape and or dtype information if it doesn't already exist.
        if ( (self.dataset_shape is None) and (self.dataset_dtype is None) ):
            dataset = self.file_handle[self.dataset_path]
            #print dataset.name
            self.dataset_shape = dataset.shape
            self.dataset_dtype = dataset.dtype.type
        elif (self.dataset_shape is None):
            dataset = self.file_handle[self.dataset_path]
            self.dataset_shape = dataset.shape
        elif (self.dataset_dtype is None):
            dataset = self.file_handle[self.dataset_path]
            self.dataset_dtype = dataset.dtype.type

        # Using the shape information, determine how to reshape the axes to present the data as we wish.
        if len(self.dataset_shape) == 1:
            self.axis_order = [-1, 0, -1, -1, -1]
        if len(self.dataset_shape) == 2:
            # self.axis_order = [-1, 0, 1, -1, -1]
            self.axis_order = [-1, 1, 0, -1, -1]
        elif (len(self.dataset_shape) == 3) and (self.dataset_shape[2] <= 4):
            # self.axis_order = [-1, 0, 1, -1, 2]
            self.axis_order = [-1, 1, 0, -1, 2]
        elif len(self.dataset_shape) == 3:
            # self.axis_order = [-1, 1, 2, -1, 0]
            self.axis_order = [-1, 2, 1, -1, 0]
        elif len(self.dataset_shape) == 4:
            # self.axis_order = [-1, 1, 2, 3, 0]
            self.axis_order = [-1, 3, 2, 1, 0]
        elif len(self.dataset_shape) == 5:
            # self.axis_order = [0, 1, 2, 3, 4]
            self.axis_order = [0, 3, 2, 1, 4]
        else:
            raise Exception("Unacceptable shape provided for display. Found shape to be \"" + self.dataset_shape + "\".")

        # Construct the shape to be singleton if the axis order is irrelevant or the appropriate shape for the reordered axis.
        self.dataset_shape = tuple([1 if _ == -1 else self.dataset_shape[_] for _ in self.axis_order])

    @debugging_tools.log_call(logger)
    def numberOfChannels(self):
        return(self.dataset_shape[-1])

    @debugging_tools.log_call(logger)
    def clean_up(self):
        # Close file
        if self.file_handle is not None:
            self.file_handle.close()
            self.file_handle = None

        self.file_path = None
        self.dataset_path = None
        self.full_path = None
        self.dataset_dtype = None
        self.dataset_shape = None

    @debugging_tools.log_call(logger)
    def dtype(self):
        return(self.dataset_dtype)

    @debugging_tools.log_call(logger)
    def shape(self):
        return(self.dataset_shape)

    @debugging_tools.log_call(logger)
    def request( self, slicing ):
        if not is_pure_slicing(slicing):
            raise Exception('HDF5DataSource: slicing is not pure')

        assert(len(slicing) == len(self.dataset_shape), "Expect a slicing for a txyzc array.")

        additional_generators.reformat_slices(slicing, self.dataset_shape)

        return(HDF5DataRequest(self.file_handle, self.dataset_path, self.axis_order, self.dataset_dtype, slicing))

    @debugging_tools.log_call(logger)
    def setDirty( self, slicing):
        if not is_pure_slicing(slicing):
            raise Exception('dirty region: slicing is not pure')
        self.isDirty.emit( slicing )

    @debugging_tools.log_call(logger)
    def __eq__( self, other ):
        if other is None:
            return False

        return(self.full_path == other.full_path)

    @debugging_tools.log_call(logger)
    def __ne__( self, other ):
        if other is None:
            return True

        return(self.full_path != other.full_path)

assert issubclass(HDF5DataSource, SourceABC)


class HDF5DataRequest( object ):
    """
        Created by an HDF5DataSource to provide a way to request slices of the HDF5 file in a nice way.
<commit_msg>neuron_volumina_viewer: HDF5DataSource was not properly formatting its slices due to a missing assignment.<commit_after>__author__ = "<NAME> <<EMAIL>>"
__date__ = "$Jun 09, 2014 8:51:33AM$"


# This program is free software; you can redistribute it and/or
# modify it under the terms of the GNU General Public License
# as published by the Free Software Foundation; either version 2
# of the License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software Foundation,
# Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
#
# Copyright 2011-2014, the ilastik developers

"""High-level API.

"""


import debugging_tools

logger = debugging_tools.logging.getLogger(__name__)

import os
import collections
import itertools
import threading

import h5py
import numpy

from PyQt4.QtGui import QApplication
from PyQt4.QtCore import pyqtSignal
from PyQt4.QtCore import QObject

from volumina.multimethods import multimethod
from volumina.pixelpipeline.datasources import SourceABC, RequestABC
from volumina.pixelpipeline.datasources import is_pure_slicing
from volumina.layer import GrayscaleLayer, RGBALayer, ColortableLayer, ClickableColortableLayer, AlphaModulatedLayer
from volumina.viewer import Viewer

import HDF5_searchers
import expanded_numpy
import additional_generators



class HDF5DatasetNotFoundException( Exception ):
    """
        An exception raised when a dataset is not found in an HDF5 file.
    """
    pass

class HDF5DataSource( QObject ):
    """
        Creates a source that reads from an HDF5 dataset and shapes it in a way that Volumina can use.

        Attributes:
              file_handle(h5py.File or str):           A handle for reading the HDF5 file or the external file path.
              file_path(str):                          External path to the file
              dataset_path(str):                       Internal path to the dataset
              full_path(str):                          Both external and internal paths combined as one path
              dataset_shape(tuple of ints):            A tuple representing the shape of the dataset in each dimension
              dataset_dtype(numpy.dtype or type):      The type of the underlying dataset.
              axis_order(tuple of ints):               A tuple representing how to reshape the array before returning a request

    """

    #TODO: Reshaping should probably be some sort of lazyflow operator and thus removed from this directly.

    isDirty = pyqtSignal( object )
    numberOfChannelsChanged = pyqtSignal(int) # Never emitted

    @debugging_tools.log_call(logger)
    def __init__( self, file_handle, internal_path, shape = None, dtype = None):
        """
            Constructs an HDF5DataSource using a given file and path to the dataset. Optionally, the shape and dtype
            can be specified.

            Args:
                file_handle(h5py.File or str):          A file handle for the HDF5 file..
                internal_path(str):                     path to the dataset inside of the HDF5 file.
                shape(tuple of ints):                   shape of underlying dataset if not specified defaults to that of the dataset.
                dtype(numpy.dtype or type):             type of underlying dataset if not specified defaults to that of the dataset.
        """
        #TODO: Get rid of shape and dtype as arguments.

        super(HDF5DataSource, self).__init__()

        self.file_handle = None

        self.file_path = ""
        self.dataset_path = ""

        self.full_path = ""

        # Constructed standard shape and dtype if provided
        self.dataset_shape = tuple(shape) if shape is not None else None
        self.dataset_dtype = numpy.dtype(dtype).type if dtype is not None else None

        self.axis_order = [-1, -1, -1, -1, -1]

        # If it is a filename, get the file handle.
        if isinstance(file_handle, str):
            file_handle.rstrip("/")
            file_handle = h5py.File(file_handle)

        self.file_handle = file_handle

        self.file_path = self.file_handle.filename
        self.dataset_path = "/" + internal_path.strip("/")

        self.full_path = self.file_path + self.dataset_path

        # Check to see if the dataset exists in the file. Otherwise throw an Exception for it.
        if self.dataset_path not in self.file_handle:
            raise(HDF5DatasetNotFoundException("Could not find the path \"" + self.dataset_path + "\" in filename " + "\"" + self.file_path + "\"."))

        # Fill in the shape and or dtype information if it doesn't already exist.
        if ( (self.dataset_shape is None) and (self.dataset_dtype is None) ):
            dataset = self.file_handle[self.dataset_path]
            #print dataset.name
            self.dataset_shape = dataset.shape
            self.dataset_dtype = dataset.dtype.type
        elif (self.dataset_shape is None):
            dataset = self.file_handle[self.dataset_path]
            self.dataset_shape = dataset.shape
        elif (self.dataset_dtype is None):
            dataset = self.file_handle[self.dataset_path]
            self.dataset_dtype = dataset.dtype.type

        # Using the shape information, determine how to reshape the axes to present the data as we wish.
        if len(self.dataset_shape) == 1:
            self.axis_order = [-1, 0, -1, -1, -1]
        if len(self.dataset_shape) == 2:
            # self.axis_order = [-1, 0, 1, -1, -1]
            self.axis_order = [-1, 1, 0, -1, -1]
        elif (len(self.dataset_shape) == 3) and (self.dataset_shape[2] <= 4):
            # self.axis_order = [-1, 0, 1, -1, 2]
            self.axis_order = [-1, 1, 0, -1, 2]
        elif len(self.dataset_shape) == 3:
            # self.axis_order = [-1, 1, 2, -1, 0]
            self.axis_order = [-1, 2, 1, -1, 0]
        elif len(self.dataset_shape) == 4:
            # self.axis_order = [-1, 1, 2, 3, 0]
            self.axis_order = [-1, 3, 2, 1, 0]
        elif len(self.dataset_shape) == 5:
            # self.axis_order = [0, 1, 2, 3, 4]
            self.axis_order = [0, 3, 2, 1, 4]
        else:
            raise Exception("Unacceptable shape provided for display. Found shape to be \"" + self.dataset_shape + "\".")

        # Construct the shape to be singleton if the axis order is irrelevant or the appropriate shape for the reordered axis.
        self.dataset_shape = tuple([1 if _ == -1 else self.dataset_shape[_] for _ in self.axis_order])

    @debugging_tools.log_call(logger)
    def numberOfChannels(self):
        return(self.dataset_shape[-1])

    @debugging_tools.log_call(logger)
    def clean_up(self):
        # Close file
        if self.file_handle is not None:
            self.file_handle.close()
            self.file_handle = None

        self.file_path = None
        self.dataset_path = None
        self.full_path = None
        self.dataset_dtype = None
        self.dataset_shape = None

    @debugging_tools.log_call(logger)
    def dtype(self):
        return(self.dataset_dtype)

    @debugging_tools.log_call(logger)
    def shape(self):
        return(self.dataset_shape)

    @debugging_tools.log_call(logger)
    def request( self, slicing ):
        if not is_pure_slicing(slicing):
            raise Exception('HDF5DataSource: slicing is not pure')

        assert(len(slicing) == len(self.dataset_shape), "Expect a slicing for a txyzc array.")

        slicing = additional_generators.reformat_slices(slicing, self.dataset_shape)

        return(HDF5DataRequest(self.file_handle, self.dataset_path, self.axis_order, self.dataset_dtype, slicing))

    @debugging_tools.log_call(logger)
    def setDirty( self, slicing):
        if not is_pure_slicing(slicing):
            raise Exception('dirty region: slicing is not pure')
        self.isDirty.emit( slicing )

    @debugging_tools.log_call(logger)
    def __eq__( self, other ):
        if other is None:
            return False

        return(self.full_path == other.full_path)

    @debugging_tools.log_call(logger)
    def __ne__( self, other ):
        if other is None:
            return True

        return(self.full_path != other.full_path)

assert issubclass(HDF5DataSource, SourceABC)


class HDF5DataRequest( object ):
    """
        Created by an HDF5DataSource to provide a way to request slices of the HDF5 file in a nice way.

{'fraction_non_alphanumeric': 0.08272154459795364, 'fraction_numerical': 0.01462512283947049, 'mean_word_length': 2.4697152025671882, 'pattern_counts': {'":': 0, '<': 11, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3606683', 'n_tokens_mistral': 5112, 'n_tokens_neox': 4867, 'n_words': 1677}