<commit_before>"""
The data module provides utility classes to interface with various
data backends.
"""

from distutils.version import LooseVersion
from collections import defaultdict, Iterable
from itertools import groupby

try:
    import itertools.izip as zip
except ImportError:
    pass

import numpy as np
try:
    import pandas as pd
except ImportError:
    pd = None

import param

from .dimension import OrderedDict, Dimension
from .element import Element, NdElement
from .ndmapping import NdMapping, item_check, sorted_context
from .spaces import HoloMap
from . import util


class Columns(Element):
    """
    Columns provides a general baseclass for column based
    Element types. Through the use of utility class interfaces
    data may be supplied and stored in a range of formats.

    Data is assumed to be in a columnar data format with N
    observations and at least D columns, where D is the number
    of dimensions. Data supplied in one of the native formats
    will be retained. Alternatively the columns maybe supplied
    as a tuple or the rows as a list of tuples. If the data is
    purely numeric the data will automatically be converted to
    a numpy array, otherwise it will fall back to the specified
    data_type.

    Currently either an NdElement or a pandas DataFrame are
    supported as storage formats for heterogeneous data. An
    NdElement is a HoloViews wrapper around dictionary objects,
    which maps between the key dimensions and the value dimensions.

    The Columns class also provides various methods to transform
    the data in various ways and allows indexing and selecting
    along all dimensions.
    """

    datatype = param.List(['array', 'dictionary', 'dataframe' ], doc="""
        Defines the datatypes to be used for storage, the object will
        attempt to use each data interface in turn falling back to the
        each consecutive type in the list if the data type is not
        understood.""")

    def __init__(self, data, **kwargs):
        data, params, interface = DataColumns.initialize(type(self), data, kwargs)
        super(Columns, self).__init__(data, **params)
        self.interface = interface
        self.data = self._validate_data(self.data)


    def _validate_data(self, data):
        """
        Method that is often overridden in the implementation of
        specific Elements for validating and transforming the input
        data format.
        """
        return self.interface.validate_data(self, data)


    def __setstate__(self, state):
        """
        Restores OrderedDict based Columns objects, converting
        them to the up-to-date NdElement format.
        """
        self.__dict__ = state
        if isinstance(self.data, OrderedDict):
            self.data = NdElement(self.data, kdims=self.kdims,
                                  vdims=self.vdims, group=self.group,
                                  label=self.label)


    def closest(self, coords):
        """
        Given single or multiple samples along the first
        key dimension will return the closest actual sample
        coordinates.
        """
        if self.ndims > 1:
            NotImplementedError("Closest method currently only "
                                "implemented for 1D Elements")

        if not isinstance(coords, list): coords = [coords]
        xs = self.dimension_values(0)
        idxs = [np.argmin(np.abs(xs-coord)) for coord in coords]
        return [xs[idx] for idx in idxs] if len(coords) > 1 else xs[idxs[0]]


    def sort(self, by=[]):
        """
        Sorts the data by the values along the supplied
        dimensions.
        """
        if not by: by = self.kdims
        sorted_columns = self.interface.sort(self, by)
        return self.clone(sorted_columns)


    def range(self, dim, data_range=True):
        """
        Computes the range of values along a supplied
        dimension, taking into account the range and
        soft_range defined on the Dimension object.
        """
        dim = self.get_dimension(dim)
        if dim.range != (None, None):
            return dim.range
        elif dim in self.dimensions():
            if len(self):
                drange = self.interface.range(self, dim)
            else:
                drange = (np.NaN, np.NaN)
        if data_range:
            soft_range = [r for r in dim.soft_range if r is not None]
            if soft_range:
                return util.max_range([drange, soft_range])
            else:
                return drange
        else:
            return dim.soft_range


    def add_dimension(self, dimension, dim_pos, dim_val, **kwargs):
        """
        Create a new object with an additional key dimensions.
        Requires the dimension name or object, the desired position
        in the key dimensions and a key value scalar or sequence of
        the same length as the existing keys.
        """
        if isinstance(dimension, str):
            dimension = Dimension(dimension)

        if dimension.name in self.kdims:
            raise Exception('{dim} dimension already defined'.format(dim=dimension.name))

        dimensions = self.kdims[:]
        dimensions.insert(dim_pos, dimension)

        data = self.interface.add_dimension(self, dimension, dim_pos, dim_val)
        return self.clone(data, kdims=dimensions)


    def select(self, selection_specs=None, **selection):
        """
        Allows selecting data by the slices, sets and scalar
        values along a particular dimension. The indices
        should be supplied as keywords mapping between
        the selected dimension and value. Additionally
        selection_specs (taking the form of a list of
        type.group.label strings, types or functions) may
        be supplied, which will ensure the selection is
        only applied if the specs match the selected object.
        """
        if selection_specs and not self.matches(selection_specs):
            return self

        data = self.interface.select(self, **selection)
        if np.isscalar(data):
            return data
        else:
            return self.clone(data)


    def reindex(self, kdims=None, vdims=None):
        """
        Create a new object with a re-ordered set of dimensions.
        Allows converting key dimensions to value dimensions
        and vice versa.
        """
        if kdims is None:
            key_dims = [d for d in self.kdims
                        if d not in vdims]
        else:
            key_dims = [self.get_dimension(k) for k in kdims]

        if vdims is None:
            val_dims = [d for d in self.vdims
                        if d not in kdims]
        else:
            val_dims = [self.get_dimension(v) for v in vdims]


        data = self.interface.reindex(self, key_dims, val_dims)
        return self.clone(data, kdims=key_dims, vdims=val_dims)


    def __getitem__(self, slices):
        """
        Allows slicing and selecting values in the Columns object.
        Supports multiple indexing modes:

           (1) Slicing and indexing along the values of each
               dimension in the columns object using either
               scalars, slices or sets of values.
           (2) Supplying the name of a dimension as the first
               argument will return the values along that
               dimension as a numpy array.<commit_msg>Minor formatting changes<commit_after>"""
The data module provides utility classes to interface with various
data backends.
"""

from distutils.version import LooseVersion
from collections import defaultdict, Iterable
from itertools import groupby

try:
    import itertools.izip as zip
except ImportError:
    pass

import numpy as np
try:
    import pandas as pd
except ImportError:
    pd = None

import param

from .dimension import OrderedDict, Dimension
from .element import Element, NdElement
from .ndmapping import NdMapping, item_check, sorted_context
from .spaces import HoloMap
from . import util


class Columns(Element):
    """
    Columns provides a general baseclass for column based
    Element types. Through the use of utility class interfaces
    data may be supplied and stored in a range of formats.

    Data is assumed to be in a columnar data format with N
    observations and at least D columns, where D is the number
    of dimensions. Data supplied in one of the native formats
    will be retained. Alternatively the columns maybe supplied
    as a tuple or the rows as a list of tuples. If the data is
    purely numeric the data will automatically be converted to
    a numpy array, otherwise it will fall back to the specified
    data_type.

    Currently either an NdElement or a pandas DataFrame are
    supported as storage formats for heterogeneous data. An
    NdElement is a HoloViews wrapper around dictionary objects,
    which maps between the key dimensions and the value dimensions.

    The Columns class also provides various methods to transform
    the data in various ways and allows indexing and selecting
    along all dimensions.
    """

    datatype = param.List(['array', 'dictionary', 'dataframe' ], doc="""
        Defines the datatypes to be used for storage, the object will
        attempt to use each data interface in turn falling back to the
        each consecutive type in the list if the data type is not
        understood.""")

    def __init__(self, data, **kwargs):
        data, params, interface = DataColumns.initialize(type(self), data, kwargs)
        super(Columns, self).__init__(data, **params)
        self.interface = interface
        self.data = self._validate_data(self.data)


    def _validate_data(self, data):
        """
        Method that is often overridden in the implementation of
        specific Elements for validating and transforming the input
        data format.
        """
        return self.interface.validate_data(self, data)


    def __setstate__(self, state):
        """
        Restores OrderedDict based Columns objects, converting
        them to the up-to-date NdElement format.
        """
        self.__dict__ = state
        if isinstance(self.data, OrderedDict):
            self.data = NdElement(self.data, kdims=self.kdims,
                                  vdims=self.vdims, group=self.group,
                                  label=self.label)


    def closest(self, coords):
        """
        Given single or multiple samples along the first
        key dimension will return the closest actual sample
        coordinates.
        """
        if self.ndims > 1:
            NotImplementedError("Closest method currently only "
                                "implemented for 1D Elements")

        if not isinstance(coords, list): coords = [coords]
        xs = self.dimension_values(0)
        idxs = [np.argmin(np.abs(xs-coord)) for coord in coords]
        return [xs[idx] for idx in idxs] if len(coords) > 1 else xs[idxs[0]]


    def sort(self, by=[]):
        """
        Sorts the data by the values along the supplied
        dimensions.
        """
        if not by: by = self.kdims
        sorted_columns = self.interface.sort(self, by)
        return self.clone(sorted_columns)


    def range(self, dim, data_range=True):
        """
        Computes the range of values along a supplied
        dimension, taking into account the range and
        soft_range defined on the Dimension object.
        """
        dim = self.get_dimension(dim)
        if dim.range != (None, None):
            return dim.range
        elif dim in self.dimensions():
            if len(self):
                drange = self.interface.range(self, dim)
            else:
                drange = (np.NaN, np.NaN)
        if data_range:
            soft_range = [r for r in dim.soft_range if r is not None]
            if soft_range:
                return util.max_range([drange, soft_range])
            else:
                return drange
        else:
            return dim.soft_range


    def add_dimension(self, dimension, dim_pos, dim_val, **kwargs):
        """
        Create a new object with an additional key dimensions.
        Requires the dimension name or object, the desired position
        in the key dimensions and a key value scalar or sequence of
        the same length as the existing keys.
        """
        if isinstance(dimension, str):
            dimension = Dimension(dimension)

        if dimension.name in self.kdims:
            raise Exception('{dim} dimension already defined'.format(dim=dimension.name))

        dimensions = self.kdims[:]
        dimensions.insert(dim_pos, dimension)

        data = self.interface.add_dimension(self, dimension, dim_pos, dim_val)
        return self.clone(data, kdims=dimensions)


    def select(self, selection_specs=None, **selection):
        """
        Allows selecting data by the slices, sets and scalar
        values along a particular dimension. The indices
        should be supplied as keywords mapping between
        the selected dimension and value. Additionally
        selection_specs (taking the form of a list of
        type.group.label strings, types or functions) may
        be supplied, which will ensure the selection is
        only applied if the specs match the selected object.
        """
        if selection_specs and not self.matches(selection_specs):
            return self

        data = self.interface.select(self, **selection)
        if np.isscalar(data):
            return data
        else:
            return self.clone(data)


    def reindex(self, kdims=None, vdims=None):
        """
        Create a new object with a re-ordered set of dimensions.
        Allows converting key dimensions to value dimensions
        and vice versa.
        """
        if kdims is None:
            key_dims = [d for d in self.kdims if d not in vdims]
        else:
            key_dims = [self.get_dimension(k) for k in kdims]

        if vdims is None:
            val_dims = [d for d in self.vdims if d not in kdims]
        else:
            val_dims = [self.get_dimension(v) for v in vdims]

        data = self.interface.reindex(self, key_dims, val_dims)
        return self.clone(data, kdims=key_dims, vdims=val_dims)


    def __getitem__(self, slices):
        """
        Allows slicing and selecting values in the Columns object.
        Supports multiple indexing modes:

           (1) Slicing and indexing along the values of each
               dimension in the columns object using either
               scalars, slices or sets of values.
           (2) Supplying the name of a dimension as the first
               argument will return the values along that
               dimension as a numpy array.
{'fraction_non_alphanumeric': 0.06435845213849287, 'fraction_numerical': 0.0009504412763068567, 'mean_word_length': 2.219187062937063, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3587937', 'n_tokens_mistral': 3839, 'n_tokens_neox': 3754, 'n_words': 1561}