<commit_before>"""
The data module provides utility classes to interface with various
data backends.
"""

from collections import defaultdict
from itertools import groupby

import numpy as np
import param

from .dimension import OrderedDict, Dimension
from .element import Element, NdElement
from .spaces import HoloMap
from . import util


class Columns(Element):

    def __init__(self, data, **kwargs):
        if 'kdims' not in kwargs:
            kwargs['kdims'] = self.kdims
        if 'vdims' not in kwargs:
            kwargs['vdims'] = self.vdims
        data, params = ColumnarData._process_data(data, **kwargs)
        super(Columns, self).__init__(data, **params)
        self.data = self._validate_data(self.data)


    def _validate_data(self, data):
        if self.interface is None:
            return data
        else:
            return self.interface.validate_data(data)


    def select(self, selection_specs=None, **selection):
        if self.interface is None:
            data = self.data.select(**selection)
        else:
            data = self.interface.select(selection) 
        if np.isscalar(data):
            return data
        else:
            return self.clone(data)
        

    @property
    def interface(self):
        if util.is_dataframe(self.data):
            return ColumnarDataFrame(self)
        elif isinstance(self.data, np.ndarray):
            return ColumnarArray(self)


    def reindex(self, kdims=None, vdims=None):
        """
        Create a new object with a re-ordered set of dimensions.
        Allows converting key dimensions to value dimensions
        and vice versa.
        """
        if self.interface is None:
            return self.data.reindex(kdims, vdims)

        if vdims is None:
            vdims = self._cached_value_names
        elif kdims is None:
            dimensions = (self._cached_index_names +
                          self._cached_value_names)
            kdims = [d for d in dimensions if d not in vdims]
        key_dims = [self.get_dimension(k) for k in kdims]
        val_dims = [self.get_dimension(v) for v in vdims]
        data = self.interface.reindex(self.data, key_dims, val_dims)
        return self.clone(data, key_dims, val_dims)


    def __getitem__(self, slices):
        """
        Implements slicing or indexing of the data by the data x-value.
        If a single element is indexed reduces the Element2D to a single
        Scatter object.
        """
        if slices is (): return self
        if not isinstance(slices, tuple): slices = (slices,)
        selection = dict(zip(self.dimensions(label=True), slices))
        if self.interface is None:
            data = self.data.select(**selection)
        else:
            data = self.interface.select(**selection) 
        return self.clone(data)


    def sample(self, samples=[]):
        """
        Allows sampling of Columns as an iterator of coordinates
        matching the key dimensions.
        """
        if self.interface is None:
            return self.clone(self.data.sample(samples))
        else:
            return self.clone(self.interface.sample(samples))


    def reduce(self, dimensions=[], function=None, **reduce_map):
        """
        Allows collapsing of Chart objects using the supplied map of
        dimensions and reduce functions.
        """
        reduce_map = self._reduce_map(dimensions, function, reduce_map)

        if len(reduce_map) > 1:
            raise ValueError("Chart Elements may only be reduced to a point.")
        dim, reduce_fn = list(reduce_map.items())[0]
        if dim in self._cached_index_names:
            reduced_data = OrderedDict(zip(self.vdims, reduce_fn(self.data[:, self.ndims:], axis=0)))
        else:
            raise Exception("Dimension %s not found in %s" % (dim, type(self).__name__))
        return self.clone(reduced_data)


    def groupby(self, dimensions, container_type=HoloMap, **kwargs):
        if self.interface is None:
            return self.data.groupby(dimensions, container_type, **kwargs)
        else:
            return self.interface.groupby(dimensions, container_type, **kwargs)


    def __len__(self):
        if self.interface is None:
            return len(self.data)
        else:
            return len(self.interface)


    @property
    def shape(self):
        if self.interface is None:
            return (len(self), len(self.dimensions()))
        else:
            return self.interface.shape


    def dimension_values(self, dim):
        if self.interface is None:
            return self.data.dimension_values(dim)
        else:
            return self.interface.values(dim)


    def dframe(self):
        if self.interface is None:
            return self.data.dframe()
        else:
            return self.interface.dframe()


    def array(self):
        if self.interface is None:
            dims = self._cached_index_names + self._cached_value_names
            return np.column_stack([self.dimension_values(d) for d in dims])
        else:
            return self.interface.array()




class ColumnarData(param.Parameterized):

    def __init__(self, element, **params):
        self.element = element


    def array(self):
        NotImplementedError

    @property
    def ndims(self):
        self.element.ndims


    @property
    def shape(self):
        return self.element.data.shape


    @classmethod
    def _process_data(cls, data, **kwargs):
        params = {}
        if isinstance(data, NdElement):
            params['kdims'] = data.kdims
            params['vdims'] = data.vdims
            params['label'] = data.label
        elif isinstance(data, Element):
            params = dict(data.get_param_values(onlychanged=True))
            data = data.data
        elif util.is_dataframe(data):
            kdims, vdims = cls._process_df_dims(data, params)
            params['kdims'] = kdims
            params['vdims'] = vdims
        elif isinstance(data, tuple):
            data = np.column_stack(data)
        elif not isinstance(data, (np.ndarray, dict)):
            data = np.array() if data is None else list(data)
            if all(np.isscalar(d) for coord in data for d in coord):
                data = np.array(data)
            elif len(data):
                data = OrderedDict(data)
        params.update(kwargs)
        if isinstance(data, dict):
            data = NdElement(data, kdims=params['kdims'],
                             vdims=params['vdims'])
        return data, params<commit_msg>Added general mechanism for Element casting to Columns type<commit_after>"""
The data module provides utility classes to interface with various
data backends.
"""

from collections import defaultdict
from itertools import groupby

import numpy as np
try:
    import pandas as pd
except ImportError:
    pd = None

import param

from .dimension import OrderedDict, Dimension
from .element import Element, NdElement
from .spaces import HoloMap
from . import util


class Columns(Element):

    def __init__(self, data, **kwargs):
        data, params = ColumnarData._process_data(data, **kwargs)
        if 'kdims' not in params:
            params['kdims'] = self.kdims
        if 'vdims' not in params:
            params['vdims'] = self.vdims
        super(Columns, self).__init__(data, **params)
        self.data = self._validate_data(self.data)


    def _validate_data(self, data):
        if self.interface is None:
            return data
        else:
            return self.interface.validate_data(data)


    def select(self, selection_specs=None, **selection):
        if self.interface is None:
            data = self.data.select(**selection)
        else:
            data = self.interface.select(selection) 
        if np.isscalar(data):
            return data
        else:
            return self.clone(data)
        

    @property
    def interface(self):
        if util.is_dataframe(self.data):
            return ColumnarDataFrame(self)
        elif isinstance(self.data, np.ndarray):
            return ColumnarArray(self)


    def reindex(self, kdims=None, vdims=None):
        """
        Create a new object with a re-ordered set of dimensions.
        Allows converting key dimensions to value dimensions
        and vice versa.
        """
        if self.interface is None:
            return self.data.reindex(kdims, vdims)

        if vdims is None:
            vdims = self._cached_value_names
        elif kdims is None:
            dimensions = (self._cached_index_names +
                          self._cached_value_names)
            kdims = [d for d in dimensions if d not in vdims]
        key_dims = [self.get_dimension(k) for k in kdims]
        val_dims = [self.get_dimension(v) for v in vdims]
        data = self.interface.reindex(self.data, key_dims, val_dims)
        return self.clone(data, key_dims, val_dims)


    def __getitem__(self, slices):
        """
        Implements slicing or indexing of the data by the data x-value.
        If a single element is indexed reduces the Element2D to a single
        Scatter object.
        """
        if slices is (): return self
        if not isinstance(slices, tuple): slices = (slices,)
        selection = dict(zip(self.dimensions(label=True), slices))
        if self.interface is None:
            data = self.data.select(**selection)
        else:
            data = self.interface.select(**selection) 
        return self.clone(data)


    def sample(self, samples=[]):
        """
        Allows sampling of Columns as an iterator of coordinates
        matching the key dimensions.
        """
        if self.interface is None:
            return self.clone(self.data.sample(samples))
        else:
            return self.clone(self.interface.sample(samples))


    def reduce(self, dimensions=[], function=None, **reduce_map):
        """
        Allows collapsing of Chart objects using the supplied map of
        dimensions and reduce functions.
        """
        reduce_map = self._reduce_map(dimensions, function, reduce_map)

        if len(reduce_map) > 1:
            raise ValueError("Chart Elements may only be reduced to a point.")
        dim, reduce_fn = list(reduce_map.items())[0]
        if dim in self._cached_index_names:
            reduced_data = OrderedDict(zip(self.vdims, reduce_fn(self.data[:, self.ndims:], axis=0)))
        else:
            raise Exception("Dimension %s not found in %s" % (dim, type(self).__name__))
        return self.clone(reduced_data)


    def groupby(self, dimensions, container_type=HoloMap, **kwargs):
        if self.interface is None:
            return self.data.groupby(dimensions, container_type, **kwargs)
        else:
            return self.interface.groupby(dimensions, container_type, **kwargs)


    def __len__(self):
        if self.interface is None:
            return len(self.data)
        else:
            return len(self.interface)


    @property
    def shape(self):
        if self.interface is None:
            return (len(self), len(self.dimensions()))
        else:
            return self.interface.shape


    def dimension_values(self, dim):
        if self.interface is None:
            return self.data.dimension_values(dim)
        else:
            return self.interface.values(dim)


    def dframe(self):
        if self.interface is None:
            return self.data.dframe()
        else:
            return self.interface.dframe()


    def array(self):
        if self.interface is None:
            dims = self._cached_index_names + self._cached_value_names
            return np.column_stack([self.dimension_values(d) for d in dims])
        else:
            return self.interface.array()




class ColumnarData(param.Parameterized):

    def __init__(self, element, **params):
        self.element = element


    def array(self):
        NotImplementedError

    @property
    def ndims(self):
        self.element.ndims


    @property
    def shape(self):
        return self.element.data.shape


    @classmethod
    def _process_data(cls, data, **kwargs):
        params = {}
        if isinstance(data, Element):
            params['kdims'] = data.kdims
            params['vdims'] = data.vdims
            params['label'] = data.label
            if data.group != data.params()['group'].default:
                params['group'] = data.group

        if isinstance(data, NdElement):
            pass
        elif isinstance(data, Columns):
            data = data.data
        elif isinstance(data, Element):
            dimensions = data.dimensions(label=True)
            columns = OrderedDict([(dim, data.dimension_values(dim))
                                for dim in dimensions])
            if pd:
                data = pd.DataFrame(columns)
            else:
                data = OrderedDict([(row[:data.ndims], row[data.ndims:])
                                    for row in zip(*columns.values())])
        elif util.is_dataframe(data):
            kdims, vdims = cls._process_df_dims(data, params)
            params['kdims'] = kdims
            params['vdims'] = vdims
        elif isinstance(data, tuple):
            data = np.column_stack(data)
        elif not isinstance(data, (np.ndarray, dict)):
            data = np.array() if data is None else list(data)
            if all(np.isscalar(d) for coord in data for d in coord):
                data = np.array(data)
            elif len(data):
                data = OrderedDict(data)
        params.update(kwargs)
        if isinstance(data, dict):
            data = NdElement(data, kdims=params['kdims'],
                             vdims=params['vdims'])
        return data, params
{'fraction_non_alphanumeric': 0.09524504084014003, 'fraction_numerical': 0.0005834305717619603, 'mean_word_length': 2.0746636771300446, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2934777', 'n_tokens_mistral': 3836, 'n_tokens_neox': 3790, 'n_words': 1067}