<commit_before>import warnings
import numpy as np
import h5py
from .core_io import IO, Reader, Writer


class hdf5IO(IO):
    """Class to wrap ABC IO for hdf5 file format."""

    def __init__(self):
        """Initialize hdf5IO instance"""
        self._file_types_ = (h5py._hl.group.Group, h5py._hl.files.File)
        self._file_format_ = "hdf5"
        super().__init__()

    def open_file(self, file_name, file_mode):
        """Open hdf5 file.

        Parameters
        ----------
        file_name : str
            path to file to open
        file_mode : str
            mode used when opening file

        Returns
        -------
        hdf5 file instance

        """
        return h5py.File(file_name, file_mode)

    def sub(self, name):
        """Create subgroup or return if already exists.

        Parameters
        ----------
        name : str
            name of subgroup

        Returns
        -------
        sub : subgroup instance

        """
        try:
            return self.base.create_group(name)
        except ValueError:
            return self.base[name]
        except KeyError:
            raise RuntimeError("Cannot create sub in reader.")

    def groups(self, where=None):
        """Finds groups in location given by 'where'.

        Parameters
        ----------
        where : None or file instance

        Returns
        -------
        groups : list

        """
        loc = self.resolve_where(where)
        return list(loc.keys())


class hdf5Reader(hdf5IO, Reader):
    """Class specifying a Reader with hdf5IO."""

    def __init__(self, target):
        """Initialize hdf5Reader class.

        Parameters
        ----------
        target : str or file instance
            Path to file OR file instance to be read.

        """
        self.target = target
        self.file_mode = "r"
        self.obj_lib = {}
        super().__init__()

    def read_obj(self, obj, where=None, obj_lib=None):
        """Read object from file in group specified by where argument.

        Parameters
        ----------
        obj : python object instance
            object must have _io_attrs_ attribute to have attributes read and loaded
        where : None or file insance
            specifies where to read obj from

        """
        if obj_lib is not None:
            self.obj_lib.update(obj_lib)
        if hasattr(obj, "_object_lib_"):
            self.obj_lib.update(obj._object_lib_)

        loc = self.resolve_where(where)
        for attr in obj._io_attrs_:
            if attr not in loc.keys():
                warnings.warn(
                    "Save attribute '{}' was not loaded.".format(attr), RuntimeWarning
                )
                continue
            if isinstance(loc[attr], h5py.Dataset):
                if isinstance(loc[attr][()], bytes):
                    setattr(obj, attr, loc[attr][()].decode("utf-8"))
                else:
                    setattr(obj, attr, loc[attr][()])
            elif isinstance(loc[attr], h5py.Group):
                if "name" not in loc[attr].keys():
                    warnings.warn(
                        "Could not load attribute '{}', no name found.".format(attr),
                        RuntimeWarning,
                    )
                    continue
                name = loc[attr]["name"][()].decode("utf-8")
                if name == "list":
                    setattr(obj, attr, self.read_list(where=loc[attr]))
                elif name == "dict":
                    setattr(obj, attr, self.read_dict(where=loc[attr]))
                else:
                    if name not in self.obj_lib:
                        warnings.warn(
                            "No object_lib '{}'.".format(name), RuntimeWarning
                        )
                        continue

                    # initialized an object from object_lib
                    setattr(
                        obj,
                        attr,
                        self.obj_lib[name].load(
                            load_from=loc[attr],
                            file_format=self._file_format_,
                            obj_lib=self.obj_lib,
                        ),
                    )

    def read_dict(self, where=None):
        """Read dictionary from file in group specified by where argument.

        Parameters
        ----------
        where : None or file instance
            specifies where to read dict from

        """
        thedict = {}
        loc = self.resolve_where(where)
        for key in loc.keys():
            if isinstance(loc[key], h5py.Dataset):
                if (
                    isinstance(loc[key][()], bytes)
                    and loc[key][()].decode("utf-8") != "name"
                ):
                    thedict[key] = loc[key][()].decode("utf-8")
                elif not isinstance(loc[key][()], bytes):
                    thedict[key] = loc[key][()]

            elif isinstance(loc[key], h5py.Group):
                if "name" not in loc[key].keys():
                    warnings.warn(
                        "Could not load attribute '{}', no name found.".format(key),
                        RuntimeWarning,
                    )
                    continue
                name = loc[key]["name"][()].decode("utf-8")
                if name == "list":
                    thedict[name] = self.read_list(where=loc[key])
                elif name == "dict":
                    thedict[key] = self.read_dict(where=loc[key])
                else:<commit_msg>Fix bug in hdf5_io<commit_after>import warnings
import numpy as np
import h5py
from .core_io import IO, Reader, Writer


class hdf5IO(IO):
    """Class to wrap ABC IO for hdf5 file format."""

    def __init__(self):
        """Initialize hdf5IO instance"""
        self._file_types_ = (h5py._hl.group.Group, h5py._hl.files.File)
        self._file_format_ = "hdf5"
        super().__init__()

    def open_file(self, file_name, file_mode):
        """Open hdf5 file.

        Parameters
        ----------
        file_name : str
            path to file to open
        file_mode : str
            mode used when opening file

        Returns
        -------
        hdf5 file instance

        """
        return h5py.File(file_name, file_mode)

    def sub(self, name):
        """Create subgroup or return if already exists.

        Parameters
        ----------
        name : str
            name of subgroup

        Returns
        -------
        sub : subgroup instance

        """
        try:
            return self.base.create_group(name)
        except ValueError:
            return self.base[name]
        except KeyError:
            raise RuntimeError("Cannot create sub in reader.")

    def groups(self, where=None):
        """Finds groups in location given by 'where'.

        Parameters
        ----------
        where : None or file instance

        Returns
        -------
        groups : list

        """
        loc = self.resolve_where(where)
        return list(loc.keys())


class hdf5Reader(hdf5IO, Reader):
    """Class specifying a Reader with hdf5IO."""

    def __init__(self, target):
        """Initialize hdf5Reader class.

        Parameters
        ----------
        target : str or file instance
            Path to file OR file instance to be read.

        """
        self.target = target
        self.file_mode = "r"
        self.obj_lib = {}
        super().__init__()

    def read_obj(self, obj, where=None, obj_lib=None):
        """Read object from file in group specified by where argument.

        Parameters
        ----------
        obj : python object instance
            object must have _io_attrs_ attribute to have attributes read and loaded
        where : None or file insance
            specifies where to read obj from

        """
        if obj_lib is not None:
            self.obj_lib.update(obj_lib)
        if hasattr(obj, "_object_lib_"):
            self.obj_lib.update(obj._object_lib_)

        loc = self.resolve_where(where)
        for attr in obj._io_attrs_:
            if attr not in loc.keys():
                warnings.warn(
                    "Save attribute '{}' was not loaded.".format(attr), RuntimeWarning
                )
                continue
            if isinstance(loc[attr], h5py.Dataset):
                if isinstance(loc[attr][()], bytes):
                    setattr(obj, attr, loc[attr][()].decode("utf-8"))
                else:
                    setattr(obj, attr, loc[attr][()])
            elif isinstance(loc[attr], h5py.Group):
                if "name" not in loc[attr].keys():
                    warnings.warn(
                        "Could not load attribute '{}', no name found.".format(attr),
                        RuntimeWarning,
                    )
                    continue
                name = loc[attr]["name"][()].decode("utf-8")
                if name == "list":
                    setattr(obj, attr, self.read_list(where=loc[attr]))
                elif name == "dict":
                    setattr(obj, attr, self.read_dict(where=loc[attr]))
                else:
                    if name not in self.obj_lib:
                        warnings.warn(
                            "No object_lib '{}'.".format(name), RuntimeWarning
                        )
                        continue

                    # initialized an object from object_lib
                    setattr(
                        obj,
                        attr,
                        self.obj_lib[name].load(
                            load_from=loc[attr],
                            file_format=self._file_format_,
                            obj_lib=self.obj_lib,
                        ),
                    )

    def read_dict(self, where=None):
        """Read dictionary from file in group specified by where argument.

        Parameters
        ----------
        where : None or file instance
            specifies where to read dict from

        """
        thedict = {}
        loc = self.resolve_where(where)
        for key in loc.keys():
            if isinstance(loc[key], h5py.Dataset):
                if isinstance(loc[key][()], bytes) and key != "name":
                    thedict[key] = loc[key][()].decode("utf-8")
                elif not isinstance(loc[key][()], bytes):
                    thedict[key] = loc[key][()]

            elif isinstance(loc[key], h5py.Group):
                if "name" not in loc[key].keys():
                    warnings.warn(
                        "Could not load attribute '{}', no name found.".format(key),
                        RuntimeWarning,
                    )
                    continue
                name = loc[key]["name"][()].decode("utf-8")
                if name == "list":
                    thedict[key] = self.read_list(where=loc[key])
                elif name == "dict":
                    thedict[key] = self.read_dict(where=loc[key])
                else:
{'fraction_non_alphanumeric': 0.11184627934378683, 'fraction_numerical': 0.004169310251064987, 'mean_word_length': 1.283526490066225, 'pattern_counts': {'":': 9, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3468087', 'n_tokens_mistral': 2896, 'n_tokens_neox': 2757, 'n_words': 760}