<commit_before>__author__ = "<NAME> <<EMAIL>>"
__date__ = "$Aug 04, 2014 14:48:56 EDT$"


import collections
import itertools
import json
import os
import os.path
import shutil
import tempfile

import numpy
import h5py

import vigra
import vigra.impex

import nanshe.nanshe.additional_generators
import nanshe.nanshe.expanded_numpy

import nanshe.nanshe.tiff_file_format
import nanshe.nanshe.nanshe_converter


class TestTiffFileFormat(object):
    def setup(self):
        self.temp_dir = ""
        self.filedata = collections.OrderedDict()
        self.data = None

        self.data = numpy.random.random_integers(0, 255, (1000, 1, 102, 101, 1)).astype(numpy.uint8)

        self.temp_dir = tempfile.mkdtemp()
        for i, i_str, (a_b, a_e) in nanshe.nanshe.additional_generators.filled_stringify_enumerate(
                                        itertools.izip(
                                                *nanshe.nanshe.additional_generators.lagged_generators(
                                                    xrange(0, self.data.shape[0] + 100 - 1, 100)
                                                )
                                        )
                                    ):
            each_filename = os.path.join(self.temp_dir, "test_tiff_" + str(i) + ".tif")
            each_data = self.data[a_b:a_e]

            self.filedata[each_filename] = each_data

            vigra.impex.writeVolume(nanshe.nanshe.expanded_numpy.tagging_reorder_array(each_data, to_axis_order="czyxt")[0, 0],
                                    os.path.join(self.temp_dir, "test_tiff_" + str(i) + ".tif"), "")

    def test_get_multipage_tiff_shape_dtype(self):
        for each_filename, each_filedata in self.filedata.items():
            each_shape_dtype = nanshe.nanshe.tiff_file_format.get_multipage_tiff_shape_dtype(each_filename)

            each_filedata = nanshe.nanshe.expanded_numpy.tagging_reorder_array(each_filedata, to_axis_order="zyxtc")[0]

            assert(each_shape_dtype["shape"] == each_filedata.shape)
            assert(each_shape_dtype["dtype"] == each_filedata.dtype.type)

    def test_get_multipage_tiff_shape_dtype_transformed(self):
        for each_filename, each_filedata in self.filedata.items():
            each_shape_dtype = nanshe.nanshe.tiff_file_format.get_multipage_tiff_shape_dtype_transformed(each_filename,
                                                                                                  axis_order = "tzyxc")

            assert(each_shape_dtype["shape"] == each_filedata.shape)
            assert(each_shape_dtype["dtype"] == each_filedata.dtype.type)

    def test_get_standard_tiff_array(self):
        for each_filename, each_filedata in self.filedata.items():
            each_data = nanshe.nanshe.tiff_file_format.get_standard_tiff_array(each_filename)

            assert(each_data.shape == each_filedata.shape)
            assert(each_data.dtype == each_filedata.dtype)

            assert((each_data == each_filedata).all())

    def test_convert_tiffs(self):
        hdf5_filename = os.path.join(self.temp_dir, "test.h5")
        hdf5_filepath = hdf5_filename + "/data"

        nanshe.nanshe.tiff_file_format.convert_tiffs(self.filedata.keys(), hdf5_filepath)

        assert(os.path.exists(hdf5_filename))

        data = None
        with h5py.File(hdf5_filename, "r") as hdf5_handle:
            data = hdf5_handle["data"].value

        self_data_h5 = nanshe.nanshe.expanded_numpy.tagging_reorder_array(self.data, to_axis_order="cztyx")[0, 0]

        assert((data == self_data_h5).all())

        os.remove(hdf5_filename)

    def test_nanshe_converter(self):
        params = {
            "axis" : 0,
            "channel" : 0,
            "z_index" : 0,
            "pages_to_channel" : 1
        }

        config_filename = os.path.join(self.temp_dir, "config.json")

        hdf5_filename = os.path.join(self.temp_dir, "test.h5")
        hdf5_filepath = hdf5_filename + "/data"

        with open(config_filename, "w") as fid:
            json.dump(params, fid)
            fid.write("\n")

        main_args = ["./nanshe_converter.py"] + ["tiff"] + [config_filename] + self.filedata.keys() + [hdf5_filepath]

        assert(nanshe.nanshe.nanshe_converter.main(*main_args) == 0)

        assert(os.path.exists(hdf5_filename))

        data = None
        with h5py.File(hdf5_filename, "r") as hdf5_handle:
            data = hdf5_handle["data"].value

        self_data_h5 = nanshe.nanshe.expanded_numpy.tagging_reorder_array(self.data, to_axis_order="cztyx")[0, 0]

        assert((data == self_data_h5).all())

        os.remove(hdf5_filename)

    def teardown(self):
        shutil.rmtree(self.temp_dir)

        self.temp_dir = ""
        self.filedata = collections.OrderedDict()
        self.data = None<commit_msg>tests/test_tiff_file_format.py: Cleanup assertions.<commit_after>__author__ = "<NAME> <<EMAIL>>"
__date__ = "$Aug 04, 2014 14:48:56 EDT$"


import collections
import itertools
import json
import os
import os.path
import shutil
import tempfile

import numpy
import h5py

import vigra
import vigra.impex

import nanshe.nanshe.additional_generators
import nanshe.nanshe.expanded_numpy

import nanshe.nanshe.tiff_file_format
import nanshe.nanshe.nanshe_converter


class TestTiffFileFormat(object):
    def setup(self):
        self.temp_dir = ""
        self.filedata = collections.OrderedDict()
        self.data = None

        self.data = numpy.random.random_integers(0, 255, (1000, 1, 102, 101, 1)).astype(numpy.uint8)

        self.temp_dir = tempfile.mkdtemp()
        for i, i_str, (a_b, a_e) in nanshe.nanshe.additional_generators.filled_stringify_enumerate(
                                        itertools.izip(
                                                *nanshe.nanshe.additional_generators.lagged_generators(
                                                    xrange(0, self.data.shape[0] + 100 - 1, 100)
                                                )
                                        )
                                    ):
            each_filename = os.path.join(self.temp_dir, "test_tiff_" + str(i) + ".tif")
            each_data = self.data[a_b:a_e]

            self.filedata[each_filename] = each_data

            vigra.impex.writeVolume(nanshe.nanshe.expanded_numpy.tagging_reorder_array(each_data, to_axis_order="czyxt")[0, 0],
                                    os.path.join(self.temp_dir, "test_tiff_" + str(i) + ".tif"), "")

    def test_get_multipage_tiff_shape_dtype(self):
        for each_filename, each_filedata in self.filedata.items():
            each_shape_dtype = nanshe.nanshe.tiff_file_format.get_multipage_tiff_shape_dtype(each_filename)

            each_filedata = nanshe.nanshe.expanded_numpy.tagging_reorder_array(each_filedata, to_axis_order="zyxtc")[0]

            assert (each_shape_dtype["shape"] == each_filedata.shape)
            assert (each_shape_dtype["dtype"] == each_filedata.dtype.type)

    def test_get_multipage_tiff_shape_dtype_transformed(self):
        for each_filename, each_filedata in self.filedata.items():
            each_shape_dtype = nanshe.nanshe.tiff_file_format.get_multipage_tiff_shape_dtype_transformed(each_filename,
                                                                                                  axis_order = "tzyxc")

            assert (each_shape_dtype["shape"] == each_filedata.shape)
            assert (each_shape_dtype["dtype"] == each_filedata.dtype.type)

    def test_get_standard_tiff_array(self):
        for each_filename, each_filedata in self.filedata.items():
            each_data = nanshe.nanshe.tiff_file_format.get_standard_tiff_array(each_filename)

            assert (each_data.shape == each_filedata.shape)
            assert (each_data.dtype == each_filedata.dtype)

            assert (each_data == each_filedata).all()

    def test_convert_tiffs(self):
        hdf5_filename = os.path.join(self.temp_dir, "test.h5")
        hdf5_filepath = hdf5_filename + "/data"

        nanshe.nanshe.tiff_file_format.convert_tiffs(self.filedata.keys(), hdf5_filepath)

        assert os.path.exists(hdf5_filename)

        data = None
        with h5py.File(hdf5_filename, "r") as hdf5_handle:
            data = hdf5_handle["data"].value

        self_data_h5 = nanshe.nanshe.expanded_numpy.tagging_reorder_array(self.data, to_axis_order="cztyx")[0, 0]

        assert (data == self_data_h5).all()

        os.remove(hdf5_filename)

    def test_nanshe_converter(self):
        params = {
            "axis" : 0,
            "channel" : 0,
            "z_index" : 0,
            "pages_to_channel" : 1
        }

        config_filename = os.path.join(self.temp_dir, "config.json")

        hdf5_filename = os.path.join(self.temp_dir, "test.h5")
        hdf5_filepath = hdf5_filename + "/data"

        with open(config_filename, "w") as fid:
            json.dump(params, fid)
            fid.write("\n")

        main_args = ["./nanshe_converter.py"] + ["tiff"] + [config_filename] + self.filedata.keys() + [hdf5_filepath]

        assert (nanshe.nanshe.nanshe_converter.main(*main_args) == 0)

        assert os.path.exists(hdf5_filename)

        data = None
        with h5py.File(hdf5_filename, "r") as hdf5_handle:
            data = hdf5_handle["data"].value

        self_data_h5 = nanshe.nanshe.expanded_numpy.tagging_reorder_array(self.data, to_axis_order="cztyx")[0, 0]

        assert (data == self_data_h5).all()

        os.remove(hdf5_filename)

    def teardown(self):
        shutil.rmtree(self.temp_dir)

        self.temp_dir = ""
        self.filedata = collections.OrderedDict()
        self.data = None
{'fraction_non_alphanumeric': 0.10568769389865564, 'fraction_numerical': 0.01592554291623578, 'mean_word_length': 2.3210851648351647, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 28, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5262379', 'n_tokens_mistral': 3347, 'n_tokens_neox': 3189, 'n_words': 457}