<commit_before>"""
Annotated Data
"""
from collections import Mapping, Sequence
from collections import OrderedDict
from enum import Enum
import numpy as np
from numpy import ma
from numpy.lib.recfunctions import append_fields
from scipy import sparse as sp
from scipy.sparse.sputils import IndexMixin
from ..utils import merge_dicts

class StorageType(Enum):
    Array = np.ndarray
    Masked = ma.MaskedArray
    Sparse = sp.spmatrix

    @classmethod
    def classes(cls):
        return tuple(c.value for c in cls.__members__.values())

SMP_NAMES = 'smp_names'
VAR_NAMES = 'var_names'

class BoundRecArr(np.recarray):
    """
    A np.recarray that can be constructed from a dict.

    Is bound to AnnData to allow adding fields.

    The column with name "index_name" plays a role analogous to the `index` in
    the `pd.Series` class.

    Attributes
    ----------
    _index_name : str
        Either SMP_NAMES or VAR_NAMES.
    _parent : AnnData
        The reference to an AnnData object to which the array is bound.
    """

    def __new__(cls, source, index_name, parent, n_row=None):
        if source is None:  # empty array
            cols = [np.arange(n_row)]
            dtype = [(name_col, 'str')]
        elif isinstance(source, np.recarray):
            cols = [source[n] for n in source.dtype.names]
            dtype = source.dtype
        else:
            if not isinstance(source, Mapping):
                raise ValueError(
                    'meta needs to be a recarray or dictlike, not {}'
                    .format(type(source)))
            # meta is dict-like
            names = list(source.keys())
            cols = [np.asarray(col) for col in source.values()]
            if name_col not in source:
                names.append(name_col)
                cols.append(np.arange(len(cols[0]) if cols else n_row).astype(str))
            dtype = list(zip(names, [str(c.dtype) for c in cols]))
        try:
            dtype = np.dtype(dtype)
        except TypeError:
            # TODO: fix compat with Python 2
            # print(dtype, file=sys.stderr)
            raise

        arr = np.recarray.__new__(cls, (len(cols[0]),), dtype)
        arr._parent = parent  # add parent as attribute
        arr._index_name = index_name  # add index_name as attribute

        for i, name in enumerate(dtype.names):
            arr[name] = np.array(cols[i], dtype=dtype[name])

        return arr

    def flipped(self):
        old_index_name = self._index_name
        new_index_name = SMP_NAMES if old_index_name == VAR_NAMES else VAR_NAMES

        flipped = BoundRecArr(self, new_index_name, self._parent, len(self))
        flipped.dtype.names = tuple(
            new_index_name if n == old_index_name else n
            for n in self.dtype.names)

        return flipped

    def copy(self):
        new = super(BoundRecArr, self).copy()
        new._index_name = self._index_name
        new._parent = self._parent
        return new

    @property
    def columns(self):
        return [c for c in self.dtype.names if not c == self._index_name]

    def __setitem__(self, keys, values):
        if isinstance(keys, str):
            keys = [keys]
            values = [values]
        keys = np.array(keys)
        values = np.array(values)  # sequence of arrays or matrix with n_key *rows*
        if not len(keys) == len(values):
            raise ValueError('You passed {} column keys but {} arrays as columns. '
                             'If you passed a matrix instead of a sequence of arrays, try transposing it.'
                             .format(len(keys), len(values)))

        present = np.intersect1d(keys, self.dtype.names)
        absent = np.setdiff1d(keys, self.dtype.names)

        if any(present):
            for k, v in zip(present, values[np.in1d(keys, present)]):
                super(BoundRecArr, self).__setitem__(k, v)

        if any(absent):
            attr = 'smp' if self._index_name == SMP_NAMES else 'var'
            if values.shape[1] > len(self):
                raise ValueError('New column has too many entries ({} > {})'
                                 .format(values.shape[1], len(self)))
            source = append_fields(self, absent, values[np.in1d(keys, absent)],
                                   usemask=False, asrecarray=True)
            new = BoundRecArr(source, self._index_name, self._parent)
            setattr(self._parent, attr, new)

    def to_df(self):
        """Return pd.dataframe."""
        import pandas as pd
        return pd.dataframe.from_records(self, index=self._index_name)


def _check_dimensions(data, smp, var):
    n_smp, n_var = data.shape<commit_msg>fixed bug in ann_data<commit_after>"""
Annotated Data
"""
from collections import Mapping, Sequence
from collections import OrderedDict
from enum import Enum
import numpy as np
from numpy import ma
from numpy.lib.recfunctions import append_fields
from scipy import sparse as sp
from scipy.sparse.sputils import IndexMixin
from ..utils import merge_dicts

class StorageType(Enum):
    Array = np.ndarray
    Masked = ma.MaskedArray
    Sparse = sp.spmatrix

    @classmethod
    def classes(cls):
        return tuple(c.value for c in cls.__members__.values())

SMP_NAMES = 'smp_names'
VAR_NAMES = 'var_names'

class BoundRecArr(np.recarray):
    """
    A np.recarray that can be constructed from a dict.

    Is bound to AnnData to allow adding fields.

    The column with name "index_name" plays a role analogous to the `index` in
    the `pd.Series` class.

    Attributes
    ----------
    _index_name : str
        Either SMP_NAMES or VAR_NAMES.
    _parent : AnnData
        The reference to an AnnData object to which the array is bound.
    """

    def __new__(cls, source, index_name, parent, n_row=None):
        if source is None:  # empty array
            cols = [np.arange(n_row)]
            dtype = [(index_name, 'str')]
        elif isinstance(source, np.recarray):
            cols = [source[n] for n in source.dtype.names]
            dtype = source.dtype
        else:
            if not isinstance(source, Mapping):
                raise ValueError(
                    'meta needs to be a recarray or dictlike, not {}'
                    .format(type(source)))
            # meta is dict-like
            names = list(source.keys())
            cols = [np.asarray(col) for col in source.values()]
            if index_name not in source:
                names.append(index_name)
                cols.append(np.arange(len(cols[0]) if cols else n_row).astype(str))
            dtype = list(zip(names, [str(c.dtype) for c in cols]))
        try:
            dtype = np.dtype(dtype)
        except TypeError:
            # TODO: fix compat with Python 2
            # print(dtype, file=sys.stderr)
            raise

        arr = np.recarray.__new__(cls, (len(cols[0]),), dtype)
        arr._parent = parent  # add parent as attribute
        arr._index_name = index_name  # add index_name as attribute

        for i, name in enumerate(dtype.names):
            arr[name] = np.array(cols[i], dtype=dtype[name])

        return arr

    def flipped(self):
        old_index_name = self._index_name
        new_index_name = SMP_NAMES if old_index_name == VAR_NAMES else VAR_NAMES

        flipped = BoundRecArr(self, new_index_name, self._parent, len(self))
        flipped.dtype.names = tuple(
            new_index_name if n == old_index_name else n
            for n in self.dtype.names)

        return flipped

    def copy(self):
        new = super(BoundRecArr, self).copy()
        new._index_name = self._index_name
        new._parent = self._parent
        return new

    @property
    def columns(self):
        return [c for c in self.dtype.names if not c == self._index_name]

    def __setitem__(self, keys, values):
        if isinstance(keys, str):
            keys = [keys]
            values = [values]
        keys = np.array(keys)
        values = np.array(values)  # sequence of arrays or matrix with n_key *rows*
        if not len(keys) == len(values):
            raise ValueError('You passed {} column keys but {} arrays as columns. '
                             'If you passed a matrix instead of a sequence of arrays, try transposing it.'
                             .format(len(keys), len(values)))

        present = np.intersect1d(keys, self.dtype.names)
        absent = np.setdiff1d(keys, self.dtype.names)

        if any(present):
            for k, v in zip(present, values[np.in1d(keys, present)]):
                super(BoundRecArr, self).__setitem__(k, v)

        if any(absent):
            attr = 'smp' if self._index_name == SMP_NAMES else 'var'
            if values.shape[1] > len(self):
                raise ValueError('New column has too many entries ({} > {})'
                                 .format(values.shape[1], len(self)))
            source = append_fields(self, absent, values[np.in1d(keys, absent)],
                                   usemask=False, asrecarray=True)
            new = BoundRecArr(source, self._index_name, self._parent)
            setattr(self._parent, attr, new)

    def to_df(self):
        """Return pd.dataframe."""
        import pandas as pd
        return pd.DataFrame().from_records(self, index=self._index_name)


def _check_dimensions(data, smp, var):
    n_smp, n_var = data.shape
{'fraction_non_alphanumeric': 0.09857051418818007, 'fraction_numerical': 0.0019202048218476637, 'mean_word_length': 2.096103038309115, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 6, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4768393', 'n_tokens_mistral': 2786, 'n_tokens_neox': 2747, 'n_words': 822}