<commit_before>
class PandasHDFDataset(object):
    """h5py Dataset-like wrapper for pandas HDFStore."""

    def __init__(self, hdf_store, key):
        self._hdf_store = hdf_store
        self.key = key
        storer = self._hdf_store.get_storer(key)
        if isinstance(storer.shape, tuple):
            self.shape = storer.shape
        else:
            if hasattr(storer.levels, '__len__'):
                ncols = storer.ncols - len(storer.levels)
            else:
                ncols = storer.ncols
            self.shape = (storer.nrows, ncols)
            assert (self.shape[0] is not None) and (self.shape[1] is not None)

    @property
    def value(self):
        return self._hdf_store[self.key]

    @property
    def dtype(self):
        return self._hdf_store.select(self.key, start=0, stop=1).values.dtype

    def select(self, *arg, **kwargs):<commit_msg>fix get shape from fixed multiindex table<commit_after>def get_shape_from_pandas_hdf_storer(storer):
    if not storer.is_table:
        shape = storer.shape
    else:
        if storer.is_multi_index:  # TODO: test when only column is MultiIndex
            ncols = storer.ncols - len(storer.levels)
        else:
            ncols = storer.ncols
        shape = (storer.nrows, ncols)
        assert (shape[0] is not None) and (shape[1] is not None)
    return shape


class PandasHDFDataset(object):
    """h5py Dataset-like wrapper for pandas HDFStore."""

    def __init__(self, hdf_store, key):
        self._hdf_store = hdf_store
        self.key = key
        self.shape = get_shape_from_pandas_hdf_storer(hdf_store.get_storer(key))

    @property
    def value(self):
        return self._hdf_store[self.key]

    @property
    def dtype(self):
        return self._hdf_store.select(self.key, start=0, stop=1).values.dtype

    def select(self, *arg, **kwargs):
{'fraction_non_alphanumeric': 0.10546241211465657, 'fraction_numerical': 0.005408328826392645, 'mean_word_length': 2.4644194756554305, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6677440', 'n_tokens_mistral': 624, 'n_tokens_neox': 611, 'n_words': 140}