<commit_before>from .. import settings
from .. import logging as logg
from .. import AnnData
from ..preprocessing.moments import get_connectivities
from ..tools.utils import strings_to_categoricals
from . import palettes

import os
import numpy as np
import pandas as pd
import matplotlib.pyplot as pl
from matplotlib.ticker import MaxNLocator
from mpl_toolkits.axes_grid1.inset_locator import inset_axes
from matplotlib.colors import is_color_like, ListedColormap, to_rgb, cnames
from matplotlib.collections import LineCollection
from matplotlib.gridspec import SubplotSpec
import matplotlib.transforms as tx
from matplotlib import rcParams
from pandas import unique, Index
from scipy.sparse import issparse
from scipy.stats import pearsonr
from cycler import Cycler, cycler
from collections import abc


"""helper functions"""


def make_dense(X):
    XA = X.A if issparse(X) and X.ndim == 2 else X.A1 if issparse(X) or isinstance(X, np.matrix) else X
    return np.array(XA)


def is_view(adata):
    return adata.is_view if hasattr(adata, 'is_view') \
        else adata.isview if hasattr(adata, 'isview') \
        else adata._isview if hasattr(adata, '_isview') else True


def is_categorical(adata, c=None):
    from pandas.api.types import is_categorical as cat
    if c is None: return cat(adata)
    if not is_view(adata): strings_to_categoricals(adata)
    return isinstance(c, str) and c in adata.obs.keys() and cat(adata.obs[c])


def is_list(key):
    return isinstance(key, (list, tuple, np.record))


def is_list_of_str(key):
    return isinstance(key, (list, tuple, np.record)) and all(isinstance(item, str) for item in key)


def is_list_of_list(lst):
    return lst is not None and any(isinstance(l, list) for l in lst)


def is_list_of_int(lst):
    return isinstance(lst, (list, tuple, np.ndarray, np.record)) and all(isinstance(item, (int, np.integer)) for item in lst)


def to_list(key, max_len=20):
    if isinstance(key, Index): key = key.tolist()
    return key if is_list(key) and len(key) < max_len else [key]


def to_val(key):
    return key[0] if isinstance(key, (list, tuple)) and len(key) == 1 else key


def get_figure_params(figsize, dpi=None, ncols=1):
    figsize = rcParams['figure.figsize'] if figsize is None else figsize
    dpi = rcParams['figure.dpi'] if dpi is None else dpi
    if settings.presenter_view and figsize[0] * ncols * (dpi / 80) > 12:
        figscale = 12 / (figsize[0] * ncols)
        figsize = (figsize[0] * figscale, figsize[1] * figscale)
        dpi = min(80, dpi)
    return figsize, dpi


def get_ax(ax, show=None, figsize=None, dpi=None, projection=None):
    figsize, _ = get_figure_params(figsize)
    if ax is None:
        ax = pl.figure(None, figsize, dpi=dpi).gca(projection='3d' if projection == '3d' else None)
        if show is None: show = True
    elif isinstance(ax, SubplotSpec):
        geo = ax.get_geometry()
        if show is None: show = geo[-1] + 1 == geo[0] * geo[1]
        ax = pl.subplot(ax)
    return ax, show


def get_kwargs(kwargs, dict_new_kwargs):
    kwargs = kwargs.copy()
    kwargs.update(dict_new_kwargs)
    return kwargs


def check_basis(adata, basis):
    if basis in adata.obsm.keys() and 'X_' + basis not in adata.obsm.keys():
        adata.obsm['X_' + basis] = adata.obsm[basis]
        logg.info('Renamed', '\'' + basis + '\'', 'to convention', '\'X_' + basis + '\' (adata.obsm).')


def get_basis(adata, basis):
    if isinstance(basis, str) and basis.startswith('X_'):
        basis = basis[2:]
    check_basis(adata, basis)
    return basis


def to_valid_bases_list(adata, keys):
    if isinstance(keys, pd.DataFrame):
        keys = keys.index
    if not isinstance(keys, str):
        keys = list(np.ravel(keys))
    keys = to_list(keys, max_len=np.inf)
    if all(isinstance(item, str) for item in keys):
        for i, key in enumerate(keys):
            if key.startswith('X_'):
                keys[i] = key = key[2:]
            check_basis(adata, key)
        valid_keys = np.hstack([adata.obs.keys(), adata.var.keys(), adata.varm.keys(), adata.obsm.keys(),
                                [key[2:] for key in adata.obsm.keys()], list(adata.layers.keys())])
        keys_ = keys
        keys = [key for key in keys if key in valid_keys or key in adata.var_names]
        keys_ = [key for key in keys_ if key not in keys]
        if len(keys_) > 0:
            logg.warn(', '.join(keys_), 'not found.')
    return keys


def get_components(components=None, basis=None, projection=None):
    if components is None: components = '1,2,3' if projection == '3d' else '1,2'
    if isinstance(components, str): components = components.split(',')
    components = np.array(components).astype(int) - 1
    if 'diffmap' in basis or 'vmap' in basis: components += 1
    return components


def get_obs_vector(adata, basis, layer=None, use_raw=None):
    return adata.obs_vector(basis, layer=layer) if layer in adata.layers.keys() \
        else adata.raw.obs_vector(basis) if use_raw else adata.obs_vector(basis)


def get_groups(adata, groups, groupby=None):
    if not isinstance(groupby, str) or groupby not in adata.obs.keys():
        groupby = 'clusters' if 'clusters' in adata.obs.keys() \
            else 'louvain' if 'louvain' in adata.obs.keys() else None<commit_msg>raise warning if embedding not yet computed, addressing https://github.com/theislab/scvelo/issues/170<commit_after>from .. import settings
from .. import logging as logg
from .. import AnnData
from ..preprocessing.moments import get_connectivities
from ..tools.utils import strings_to_categoricals
from . import palettes

import os
import numpy as np
import pandas as pd
import matplotlib.pyplot as pl
from matplotlib.ticker import MaxNLocator
from mpl_toolkits.axes_grid1.inset_locator import inset_axes
from matplotlib.colors import is_color_like, ListedColormap, to_rgb, cnames
from matplotlib.collections import LineCollection
from matplotlib.gridspec import SubplotSpec
import matplotlib.transforms as tx
from matplotlib import rcParams
from pandas import unique, Index
from scipy.sparse import issparse
from scipy.stats import pearsonr
from cycler import Cycler, cycler
from collections import abc


"""helper functions"""


def make_dense(X):
    XA = X.A if issparse(X) and X.ndim == 2 else X.A1 if issparse(X) or isinstance(X, np.matrix) else X
    return np.array(XA)


def is_view(adata):
    return adata.is_view if hasattr(adata, 'is_view') \
        else adata.isview if hasattr(adata, 'isview') \
        else adata._isview if hasattr(adata, '_isview') else True


def is_categorical(adata, c=None):
    from pandas.api.types import is_categorical as cat
    if c is None: return cat(adata)
    if not is_view(adata): strings_to_categoricals(adata)
    return isinstance(c, str) and c in adata.obs.keys() and cat(adata.obs[c])


def is_list(key):
    return isinstance(key, (list, tuple, np.record))


def is_list_of_str(key):
    return isinstance(key, (list, tuple, np.record)) and all(isinstance(item, str) for item in key)


def is_list_of_list(lst):
    return lst is not None and any(isinstance(l, list) for l in lst)


def is_list_of_int(lst):
    return isinstance(lst, (list, tuple, np.ndarray, np.record)) and all(isinstance(item, (int, np.integer)) for item in lst)


def to_list(key, max_len=20):
    if isinstance(key, Index): key = key.tolist()
    return key if is_list(key) and len(key) < max_len else [key]


def to_val(key):
    return key[0] if isinstance(key, (list, tuple)) and len(key) == 1 else key


def get_figure_params(figsize, dpi=None, ncols=1):
    figsize = rcParams['figure.figsize'] if figsize is None else figsize
    dpi = rcParams['figure.dpi'] if dpi is None else dpi
    if settings.presenter_view and figsize[0] * ncols * (dpi / 80) > 12:
        figscale = 12 / (figsize[0] * ncols)
        figsize = (figsize[0] * figscale, figsize[1] * figscale)
        dpi = min(80, dpi)
    return figsize, dpi


def get_ax(ax, show=None, figsize=None, dpi=None, projection=None):
    figsize, _ = get_figure_params(figsize)
    if ax is None:
        ax = pl.figure(None, figsize, dpi=dpi).gca(projection='3d' if projection == '3d' else None)
        if show is None: show = True
    elif isinstance(ax, SubplotSpec):
        geo = ax.get_geometry()
        if show is None: show = geo[-1] + 1 == geo[0] * geo[1]
        ax = pl.subplot(ax)
    return ax, show


def get_kwargs(kwargs, dict_new_kwargs):
    kwargs = kwargs.copy()
    kwargs.update(dict_new_kwargs)
    return kwargs


def check_basis(adata, basis):
    if basis in adata.obsm.keys() and 'X_' + basis not in adata.obsm.keys():
        adata.obsm['X_' + basis] = adata.obsm[basis]
        logg.info('Renamed', '\'' + basis + '\'', 'to convention', '\'X_' + basis + '\' (adata.obsm).')


def get_basis(adata, basis):
    if isinstance(basis, str) and basis.startswith('X_'):
        basis = basis[2:]
    check_basis(adata, basis)
    return basis


def to_valid_bases_list(adata, keys):
    if isinstance(keys, pd.DataFrame):
        keys = keys.index
    if not isinstance(keys, str):
        keys = list(np.ravel(keys))
    keys = to_list(keys, max_len=np.inf)
    if all(isinstance(item, str) for item in keys):
        for i, key in enumerate(keys):
            if key.startswith('X_'):
                keys[i] = key = key[2:]
            check_basis(adata, key)
        valid_keys = np.hstack([adata.obs.keys(), adata.var.keys(), adata.varm.keys(), adata.obsm.keys(),
                                [key[2:] for key in adata.obsm.keys()], list(adata.layers.keys())])
        keys_ = keys
        keys = [key for key in keys if key in valid_keys or key in adata.var_names]
        keys_ = [key for key in keys_ if key not in keys]
        if len(keys_) > 0:
            msg_embedding = ''
            if len(keys_) == 1 and keys_[0] in {'diffmap', 'umap', 'tsne'}:
                msg_embedding = f"You need to run `scv.tl.{keys_[0]}` first."
            logg.warn(', '.join(keys_), 'not found.', msg_embedding)
    return keys


def get_components(components=None, basis=None, projection=None):
    if components is None: components = '1,2,3' if projection == '3d' else '1,2'
    if isinstance(components, str): components = components.split(',')
    components = np.array(components).astype(int) - 1
    if 'diffmap' in basis or 'vmap' in basis: components += 1
    return components


def get_obs_vector(adata, basis, layer=None, use_raw=None):
    return adata.obs_vector(basis, layer=layer) if layer in adata.layers.keys() \
        else adata.raw.obs_vector(basis) if use_raw else adata.obs_vector(basis)


def get_value_counts(adata, color):
    value_counts = adata.obs[color].value_counts()
    probs = np.array(adata.obs[color])
    for cat in value_counts.index:
        probs[probs == cat] = value_counts[cat]
    return np.array(probs, dtype=np.float32)


def get_groups(adata, groups, groupby=None):
    if not isinstance(groupby, str) or groupby not in adata.obs.keys():
        groupby = 'clusters' if 'clusters' in adata.obs.keys() \
            else 'louvain' if 'louvain' in adata.obs.keys() else None
{'fraction_non_alphanumeric': 0.11927017796153155, 'fraction_numerical': 0.007549883156570196, 'mean_word_length': 3.4102259215219974, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 7, 'https://': 1, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '889046', 'n_tokens_mistral': 3886, 'n_tokens_neox': 3757, 'n_words': 1114}