<commit_before># -*- coding: utf-8 -*-
"""
Created on Tue Nov  3 15:07:16 2017

@author: <NAME>
"""

import unittest
import os
import sys
import h5py
import numpy as np
import shutil
sys.path.append("../../../pycroscopy/")
from pycroscopy import VirtualGroup, VirtualDataset
from pycroscopy import HDFwriter
from pycroscopy.core.io import hdf_utils, write_utils
from pycroscopy.core.io.pycro_data import PycroDataset

test_h5_file_path = 'test_hdf_utils.h5'

if sys.version_info.major == 3:
    unicode = str


class TestHDFUtils(unittest.TestCase):

    @staticmethod
    def __delete_existing_file(file_path):
        if os.path.exists(file_path):
            os.remove(file_path)

    @staticmethod
    def __ensure_test_h5_file():
        if not os.path.exists(test_h5_file_path):
            TestHDFUtils.__create_test_h5_file()

    @staticmethod
    def __create_test_h5_file():
        if os.path.exists(test_h5_file_path):
            os.remove(test_h5_file_path)
        with h5py.File(test_h5_file_path) as h5_f:
            num_rows = 3
            num_cols = 5
            source_dset_name = 'source_main'
            tool_name = 'Fitter'

            source_pos_data = np.vstack((np.tile(np.arange(num_cols), num_rows),
                                         np.repeat(np.arange(num_rows), num_cols))).T
            pos_attrs = {'units': ['nm', 'um'],
                         'labels': {'X': (slice(None), slice(0, 1)), 'Y': (slice(None), slice(1, 2))}}
            dset_source_pos_inds = VirtualDataset('Position_Indices', source_pos_data, dtype=np.uint16, attrs=pos_attrs)
            # make the values more interesting:
            source_pos_data = np.vstack((source_pos_data[:, 0] * 50, source_pos_data[:, 1] * 1.25)).T
            dset_source_pos_vals = VirtualDataset('Position_Values', source_pos_data, dtype=np.float16, attrs=pos_attrs)

            num_cycles = 2
            num_cycle_pts = 7

            source_main_data = np.random.rand(num_rows * num_cols, num_cycle_pts * num_cycles)
            dset_source_main = VirtualDataset(source_dset_name, source_main_data,
                                              attrs={'units': 'A', 'quantity': 'Current',
                                                   'labels': {'even_rows': (slice(0, None, 2), slice(None)),
                                                              'odd_rows': (slice(1, None, 2), slice(None))}
                                                   })
            # make spectroscopic axis interesting as well
            source_spec_data = np.vstack((np.tile(np.arange(num_cycle_pts), num_cycles),
                                          np.repeat(np.arange(num_cycles), num_cycle_pts)))
            source_spec_attrs = {'units': ['V', ''],
                                 'labels': {'Bias': (slice(0, 1), slice(None)), 'Cycle': (slice(1, 2), slice(None))}}
            dset_source_spec_inds = VirtualDataset('Spectroscopic_Indices', source_spec_data, dtype=np.uint16,
                                                   attrs=source_spec_attrs)
            source_spec_data = np.vstack((np.tile(2.5 * np.sin(np.linspace(0, np.pi, num_cycle_pts, endpoint=False)),
                                                  num_cycles),
                                          np.repeat(np.arange(num_cycles), num_cycle_pts)))
            dset_source_spec_vals = VirtualDataset('Spectroscopic_Values', source_spec_data, dtype=np.float16,
                                                   attrs=source_spec_attrs)

            dset_ancillary = VirtualDataset('Ancillary', np.arange(5))

            group_source = VirtualGroup('Raw_Measurement',
                                        children=[dset_source_main, dset_source_spec_inds, dset_source_spec_vals,
                                                  dset_source_pos_vals, dset_source_pos_inds, dset_ancillary,
                                                  VirtualGroup('Misc')],
                                        attrs={'att_1': 'string_val', 'att_2': 1.2345, 'att_3': [1, 2, 3, 4],
                                                 'att_4': ['str_1', 'str_2', 'str_3']})

            writer = HDFwriter(h5_f)
            h5_refs_list = writer.write(group_source, verbose=False)

            [h5_source_main] = hdf_utils.get_h5_obj_refs([dset_source_main.name], h5_refs_list)
            h5_source_group = h5_source_main.parent
            [h5_pos_inds] = hdf_utils.get_h5_obj_refs([dset_source_pos_inds.name], h5_refs_list)
            [h5_pos_vals] = hdf_utils.get_h5_obj_refs([dset_source_pos_vals.name], h5_refs_list)
            [h5_source_spec_inds] = hdf_utils.get_h5_obj_refs([dset_source_spec_inds.name], h5_refs_list)
            [h5_source_spec_vals] = hdf_utils.get_h5_obj_refs([dset_source_spec_vals.name], h5_refs_list)

            # Now need to link as main!
            for dset in [h5_pos_inds, h5_pos_vals, h5_source_spec_inds, h5_source_spec_vals]:
                h5_source_main.attrs[dset.name.split('/')[-1]] = dset.ref

            # Now add a few results:

            num_cycles = 1
            num_cycle_pts = 7

            results_spec_inds = np.expand_dims(np.arange(num_cycle_pts), 0)
            results_spec_attrs = {'units': ['V'], 'labels': {'Bias': (slice(0, 1), slice(None))}}
            dset_results_spec_inds = VirtualDataset('Spectroscopic_Indices', results_spec_inds, dtype=np.uint16,
                                                    attrs=results_spec_attrs)
            results_spec_vals = np.expand_dims(2.5 * np.sin(np.linspace(0, np.pi, num_cycle_pts, endpoint=False)), 0)
            dset_results_spec_vals = VirtualDataset('Spectroscopic_Values', results_spec_vals, dtype=np.float16,
                                                    attrs=results_spec_attrs)

            results_1_main_data = np.random.rand(num_rows * num_cols, num_cycle_pts * num_cycles)
            dset_results_1_main = VirtualDataset('results_main', results_1_main_data,
                                                 attrs={'units': 'pF', 'quantity': 'Capacitance'})

            group_results_1 = VirtualGroup(source_dset_name + '-' + tool_name + '_',
                                           parent=h5_source_group.name,
                                           children=[dset_results_1_main, dset_results_spec_inds,
                                                       dset_results_spec_vals],
                                           attrs={'att_1': 'string_val', 'att_2': 1.2345,
                                                    'att_3': [1, 2, 3, 4], 'att_4': ['str_1', 'str_2', 'str_3']})

            h5_refs_list = writer.write(group_results_1, verbose=False)

            [h5_results_1_main] = hdf_utils.get_h5_obj_refs([dset_results_1_main.name], h5_refs_list)
            [h5_results_1_spec_inds] = hdf_utils.get_h5_obj_refs([dset_results_spec_inds.name], h5_refs_list)
            [h5_results_1_spec_vals] = hdf_utils.get_h5_obj_refs([dset_results_spec_vals.name], h5_refs_list)

            # Now need to link as main!
            for dset in [h5_pos_inds, h5_pos_vals, h5_results_1_spec_inds, h5_results_1_spec_vals]:
                h5_results_1_main.attrs[dset.name.split('/')[-1]] = dset.ref

            # add another result with different parameters

            results_2_main_data = np.random.rand(num_rows * num_cols, num_cycle_pts * num_cycles)
            dset_results_2_main = VirtualDataset('results_main', results_2_main_data,
                                                 attrs={'units': 'pF', 'quantity': 'Capacitance'})

            group_results_2 = VirtualGroup(source_dset_name + '-' + tool_name + '_',
                                           parent=h5_source_group.name,
                                           children=[dset_results_2_main, dset_results_spec_inds,
                                                       dset_results_spec_vals],
                                           attrs={'att_1': 'other_string_val', 'att_2': 5.4321,
                                                    'att_3': [4, 1, 3], 'att_4': ['s', 'str_2', 'str_3']})

            h5_refs_list = writer.write(group_results_2, verbose=False)

            [h5_results_2_main] = hdf_utils.get_h5_obj_refs([dset_results_2_main.name], h5_refs_list)
            [h5_results_2_spec_inds] = hdf_utils.get_h5_obj_refs([dset_results_spec_inds.name], h5_refs_list)
            [h5_results_2_spec_vals] = hdf_utils.get_h5_obj_refs([dset_results_spec_vals.name], h5_refs_list)

            # Now need to link as main!
            for dset in [h5_pos_inds, h5_pos_vals, h5_results_2_spec_inds, h5_results_2_spec_vals]:
                h5_results_2_main.attrs[dset.name.split('/')[-1]] = dset.ref

    # start off with the most popular functions:
    def test_get_attr_illegal_01(self):
        self.__ensure_test_h5_file()
        with self.assertRaises(AssertionError):
            hdf_utils.get_attr(np.arange(3), 'units')

    def test_get_attr_illegal_02(self):
        self.__ensure_test_h5_file()
        with h5py.File(test_h5_file_path, mode='r') as h5_f:<commit_msg>Fixed the accidental text replacement from print_log to verbose for the HDFWriter<commit_after># -*- coding: utf-8 -*-
"""
Created on Tue Nov  3 15:07:16 2017

@author: <NAME>
"""

import unittest
import os
import sys
import h5py
import numpy as np
import shutil
sys.path.append("../../../pycroscopy/")
from pycroscopy import VirtualGroup, VirtualDataset
from pycroscopy import HDFwriter
from pycroscopy.core.io import hdf_utils, write_utils
from pycroscopy.core.io.pycro_data import PycroDataset

test_h5_file_path = 'test_hdf_utils.h5'

if sys.version_info.major == 3:
    unicode = str


class TestHDFUtils(unittest.TestCase):

    @staticmethod
    def __delete_existing_file(file_path):
        if os.path.exists(file_path):
            os.remove(file_path)

    @staticmethod
    def __ensure_test_h5_file():
        if not os.path.exists(test_h5_file_path):
            TestHDFUtils.__create_test_h5_file()

    @staticmethod
    def __create_test_h5_file():
        if os.path.exists(test_h5_file_path):
            os.remove(test_h5_file_path)
        with h5py.File(test_h5_file_path) as h5_f:
            num_rows = 3
            num_cols = 5
            source_dset_name = 'source_main'
            tool_name = 'Fitter'

            source_pos_data = np.vstack((np.tile(np.arange(num_cols), num_rows),
                                         np.repeat(np.arange(num_rows), num_cols))).T
            pos_attrs = {'units': ['nm', 'um'],
                         'labels': {'X': (slice(None), slice(0, 1)), 'Y': (slice(None), slice(1, 2))}}
            dset_source_pos_inds = VirtualDataset('Position_Indices', source_pos_data, dtype=np.uint16, attrs=pos_attrs)
            # make the values more interesting:
            source_pos_data = np.vstack((source_pos_data[:, 0] * 50, source_pos_data[:, 1] * 1.25)).T
            dset_source_pos_vals = VirtualDataset('Position_Values', source_pos_data, dtype=np.float16, attrs=pos_attrs)

            num_cycles = 2
            num_cycle_pts = 7

            source_main_data = np.random.rand(num_rows * num_cols, num_cycle_pts * num_cycles)
            dset_source_main = VirtualDataset(source_dset_name, source_main_data,
                                              attrs={'units': 'A', 'quantity': 'Current',
                                                   'labels': {'even_rows': (slice(0, None, 2), slice(None)),
                                                              'odd_rows': (slice(1, None, 2), slice(None))}
                                                   })
            # make spectroscopic axis interesting as well
            source_spec_data = np.vstack((np.tile(np.arange(num_cycle_pts), num_cycles),
                                          np.repeat(np.arange(num_cycles), num_cycle_pts)))
            source_spec_attrs = {'units': ['V', ''],
                                 'labels': {'Bias': (slice(0, 1), slice(None)), 'Cycle': (slice(1, 2), slice(None))}}
            dset_source_spec_inds = VirtualDataset('Spectroscopic_Indices', source_spec_data, dtype=np.uint16,
                                                   attrs=source_spec_attrs)
            source_spec_data = np.vstack((np.tile(2.5 * np.sin(np.linspace(0, np.pi, num_cycle_pts, endpoint=False)),
                                                  num_cycles),
                                          np.repeat(np.arange(num_cycles), num_cycle_pts)))
            dset_source_spec_vals = VirtualDataset('Spectroscopic_Values', source_spec_data, dtype=np.float16,
                                                   attrs=source_spec_attrs)

            dset_ancillary = VirtualDataset('Ancillary', np.arange(5))

            group_source = VirtualGroup('Raw_Measurement',
                                        children=[dset_source_main, dset_source_spec_inds, dset_source_spec_vals,
                                                  dset_source_pos_vals, dset_source_pos_inds, dset_ancillary,
                                                  VirtualGroup('Misc')],
                                        attrs={'att_1': 'string_val', 'att_2': 1.2345, 'att_3': [1, 2, 3, 4],
                                                 'att_4': ['str_1', 'str_2', 'str_3']})

            writer = HDFwriter(h5_f)
            h5_refs_list = writer.write(group_source, print_log=False)

            [h5_source_main] = hdf_utils.get_h5_obj_refs([dset_source_main.name], h5_refs_list)
            h5_source_group = h5_source_main.parent
            [h5_pos_inds] = hdf_utils.get_h5_obj_refs([dset_source_pos_inds.name], h5_refs_list)
            [h5_pos_vals] = hdf_utils.get_h5_obj_refs([dset_source_pos_vals.name], h5_refs_list)
            [h5_source_spec_inds] = hdf_utils.get_h5_obj_refs([dset_source_spec_inds.name], h5_refs_list)
            [h5_source_spec_vals] = hdf_utils.get_h5_obj_refs([dset_source_spec_vals.name], h5_refs_list)

            # Now need to link as main!
            for dset in [h5_pos_inds, h5_pos_vals, h5_source_spec_inds, h5_source_spec_vals]:
                h5_source_main.attrs[dset.name.split('/')[-1]] = dset.ref

            # Now add a few results:

            num_cycles = 1
            num_cycle_pts = 7

            results_spec_inds = np.expand_dims(np.arange(num_cycle_pts), 0)
            results_spec_attrs = {'units': ['V'], 'labels': {'Bias': (slice(0, 1), slice(None))}}
            dset_results_spec_inds = VirtualDataset('Spectroscopic_Indices', results_spec_inds, dtype=np.uint16,
                                                    attrs=results_spec_attrs)
            results_spec_vals = np.expand_dims(2.5 * np.sin(np.linspace(0, np.pi, num_cycle_pts, endpoint=False)), 0)
            dset_results_spec_vals = VirtualDataset('Spectroscopic_Values', results_spec_vals, dtype=np.float16,
                                                    attrs=results_spec_attrs)

            results_1_main_data = np.random.rand(num_rows * num_cols, num_cycle_pts * num_cycles)
            dset_results_1_main = VirtualDataset('results_main', results_1_main_data,
                                                 attrs={'units': 'pF', 'quantity': 'Capacitance'})

            group_results_1 = VirtualGroup(source_dset_name + '-' + tool_name + '_',
                                           parent=h5_source_group.name,
                                           children=[dset_results_1_main, dset_results_spec_inds,
                                                       dset_results_spec_vals],
                                           attrs={'att_1': 'string_val', 'att_2': 1.2345,
                                                    'att_3': [1, 2, 3, 4], 'att_4': ['str_1', 'str_2', 'str_3']})

            h5_refs_list = writer.write(group_results_1, print_log=False)

            [h5_results_1_main] = hdf_utils.get_h5_obj_refs([dset_results_1_main.name], h5_refs_list)
            [h5_results_1_spec_inds] = hdf_utils.get_h5_obj_refs([dset_results_spec_inds.name], h5_refs_list)
            [h5_results_1_spec_vals] = hdf_utils.get_h5_obj_refs([dset_results_spec_vals.name], h5_refs_list)

            # Now need to link as main!
            for dset in [h5_pos_inds, h5_pos_vals, h5_results_1_spec_inds, h5_results_1_spec_vals]:
                h5_results_1_main.attrs[dset.name.split('/')[-1]] = dset.ref

            # add another result with different parameters

            results_2_main_data = np.random.rand(num_rows * num_cols, num_cycle_pts * num_cycles)
            dset_results_2_main = VirtualDataset('results_main', results_2_main_data,
                                                 attrs={'units': 'pF', 'quantity': 'Capacitance'})

            group_results_2 = VirtualGroup(source_dset_name + '-' + tool_name + '_',
                                           parent=h5_source_group.name,
                                           children=[dset_results_2_main, dset_results_spec_inds,
                                                       dset_results_spec_vals],
                                           attrs={'att_1': 'other_string_val', 'att_2': 5.4321,
                                                    'att_3': [4, 1, 3], 'att_4': ['s', 'str_2', 'str_3']})

            h5_refs_list = writer.write(group_results_2, print_log=False)

            [h5_results_2_main] = hdf_utils.get_h5_obj_refs([dset_results_2_main.name], h5_refs_list)
            [h5_results_2_spec_inds] = hdf_utils.get_h5_obj_refs([dset_results_spec_inds.name], h5_refs_list)
            [h5_results_2_spec_vals] = hdf_utils.get_h5_obj_refs([dset_results_spec_vals.name], h5_refs_list)

            # Now need to link as main!
            for dset in [h5_pos_inds, h5_pos_vals, h5_results_2_spec_inds, h5_results_2_spec_vals]:
                h5_results_2_main.attrs[dset.name.split('/')[-1]] = dset.ref

    # start off with the most popular functions:
    def test_get_attr_illegal_01(self):
        self.__ensure_test_h5_file()
        with self.assertRaises(AssertionError):
            hdf_utils.get_attr(np.arange(3), 'units')

    def test_get_attr_illegal_02(self):
        self.__ensure_test_h5_file()
        with h5py.File(test_h5_file_path, mode='r') as h5_f:
{'fraction_non_alphanumeric': 0.10353507715880486, 'fraction_numerical': 0.02342125424099814, 'mean_word_length': 1.989041543997383, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 114, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '904942', 'n_tokens_mistral': 5903, 'n_tokens_neox': 5681, 'n_words': 886}