<commit_before>import argparse
import enum
import os
import typing

import SimpleITK as sitk
import numpy as np

import miapy.data as miapy_data
import miapy.data.conversion as conv
import miapy.data.extraction as extr
import miapy.data.loading as miapy_load
import miapy.data.transformation as miapy_tfm
import miapy.data.creation.fileloader as file_load


def main(hdf_file: str):

    with extr.get_reader(hdf_file) as reader:
        extractor = extr.ComposeExtractor([extr.NamesExtractor(),
                                           extr.ImageExtractor(),
                                           extr.LabelExtractor(),
                                           extr.SupplementaryExtractor(('AGE', 'SEX'), entire_subject=True),
                                           extr.SupplementaryExtractor(('MASK',), entire_subject=False),
                                           extr.SubjectExtractor(),
                                           extr.FilesExtractor(),
                                           extr.IndexingExtractor(do_pickle_expression=False),
                                           extr.ImagePropertiesExtractor()])
        dataset = extr.ParameterizableDataset(reader, extr.SliceIndexing(), extractor)

        for i in range(len(dataset)):
            item = dataset[i]

            index_expr = item['index_expr']  # type: miapy_data.IndexExpression
            root = item['file_root']

            image = None  # type: sitk.Image
            for i, file in enumerate(item['image_files']):
                image = sitk.ReadImage(os.path.join(root, file))
                np_img = sitk.GetArrayFromImage(image)
                np_slice = np_img[index_expr.expression]
                np_slice = np_slice / np_slice.max()  # normalize
                if (np_slice != item['images'][..., i]).any():
                    raise ValueError('slice not equal')

            # for any image
            image_properties = conv.ImageProperties(image)

            if image_properties != item['image_properties']:
                raise ValueError('image properties not equal')

            for i, file in enumerate(item['label_files']):
                image = sitk.ReadImage(os.path.join(root, file))
                np_img = sitk.GetArrayFromImage(image)
                np_slice = np_img[index_expr.expression]
                if (np_slice != item['labels']).any():
                    raise ValueError('slice not equal')

            for i, file in enumerate(item['supplementary_files']):
                id_ = item['supplementary_names'][i]
                if id_ == 'MASK':
                    image = sitk.ReadImage(os.path.join(root, file))
                    np_img = sitk.GetArrayFromImage(image)
                    np_slice = np_img[index_expr.expression]
                    if (np_slice != item[id_]).any():
                        raise ValueError('slice not equal')
                else:
                    line = 0 if id_ == 'AGE' else 1
                    with open(os.path.join(root, file), 'r') as f:
                        value = f.readlines()[line].split(':')[1].strip()
                    if value != str(item[id_][0]):
                        raise ValueError('value not equal')


if __name__ == '__main__':
    """The program's entry point.

    Parse the arguments and run the program.
    """

    parser = argparse.ArgumentParser(description='Dataset creation')

    parser.add_argument(
        '--hdf_file',
        type=str,
        default='out/test/test.h5',<commit_msg>fix(verify_dataset): file path to ParameterizableDataset instead of reader<commit_after>import argparse
import os

import SimpleITK as sitk

import miapy.data as miapy_data
import miapy.data.conversion as conv
import miapy.data.extraction as extr


def main(hdf_file: str):
    extractor = extr.ComposeExtractor([extr.NamesExtractor(),
                                       extr.ImageExtractor(),
                                       extr.LabelExtractor(),
                                       extr.SupplementaryExtractor(('AGE', 'SEX'), entire_subject=True),
                                       extr.SupplementaryExtractor(('MASK',), entire_subject=False),
                                       extr.SubjectExtractor(),
                                       extr.FilesExtractor(),
                                       extr.IndexingExtractor(do_pickle_expression=False),
                                       extr.ImagePropertiesExtractor()])
    dataset = extr.ParameterizableDataset(hdf_file, extr.SliceIndexing(), extractor)

    for i in range(len(dataset)):
        item = dataset[i]

        index_expr = item['index_expr']  # type: miapy_data.IndexExpression
        root = item['file_root']

        image = None  # type: sitk.Image
        for i, file in enumerate(item['image_files']):
            image = sitk.ReadImage(os.path.join(root, file))
            np_img = sitk.GetArrayFromImage(image)
            np_slice = np_img[index_expr.expression]
            np_slice = np_slice / np_slice.max()  # normalize
            if (np_slice != item['images'][..., i]).any():
                raise ValueError('slice not equal')

        # for any image
        image_properties = conv.ImageProperties(image)

        if image_properties != item['image_properties']:
            raise ValueError('image properties not equal')

        for i, file in enumerate(item['label_files']):
            image = sitk.ReadImage(os.path.join(root, file))
            np_img = sitk.GetArrayFromImage(image)
            np_slice = np_img[index_expr.expression]
            if (np_slice != item['labels']).any():
                raise ValueError('slice not equal')

        for i, file in enumerate(item['supplementary_files']):
            id_ = item['supplementary_names'][i]
            if id_ == 'MASK':
                image = sitk.ReadImage(os.path.join(root, file))
                np_img = sitk.GetArrayFromImage(image)
                np_slice = np_img[index_expr.expression]
                if (np_slice != item[id_]).any():
                    raise ValueError('slice not equal')
            else:
                line = 0 if id_ == 'AGE' else 1
                with open(os.path.join(root, file), 'r') as f:
                    value = f.readlines()[line].split(':')[1].strip()
                if value != str(item[id_][0]):
                    raise ValueError('value not equal')

    print('All test passed!')

if __name__ == '__main__':
    """The program's entry point.

    Parse the arguments and run the program.
    """

    parser = argparse.ArgumentParser(description='Dataset creation')

    parser.add_argument(
        '--hdf_file',
        type=str,
        default='out/test/test.h5',
{'fraction_non_alphanumeric': 0.10625370480142264, 'fraction_numerical': 0.0014819205690574985, 'mean_word_length': 1.7750822368421053, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 24, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3848749', 'n_tokens_mistral': 1803, 'n_tokens_neox': 1741, 'n_words': 393}