<commit_before>from __future__ import absolute_import

from distutils.version import LooseVersion

try:
    import itertools.izip as zip
except ImportError:
    pass

import numpy as np
import pandas as pd

from .interface import Interface, DataError
from ..dimension import Dimension
from ..element import Element
from ..dimension import OrderedDict as cyODict
from ..ndmapping import NdMapping, item_check
from .. import util


class PandasInterface(Interface):

    types = (pd.DataFrame if pd else None,)

    datatype = 'dataframe'

    @classmethod
    def dimension_type(cls, columns, dim):
        name = columns.get_dimension(dim, strict=True).name
        idx = list(columns.data.columns).index(name)
        return columns.data.dtypes[idx].type

    @classmethod
    def init(cls, eltype, data, kdims, vdims):
        element_params = eltype.params()
        kdim_param = element_params['kdims']
        vdim_param = element_params['vdims']
        if util.is_series(data):
            data = data.to_frame()
        if util.is_dataframe(data):
            if eltype._auto_indexable_1d and len(data.columns) == 1:
                data = data.reset_index()
            if isinstance(kdim_param.bounds[1], int):
                ndim = min([kdim_param.bounds[1], len(kdim_param.default)])
            else:
                ndim = None
            nvdim = vdim_param.bounds[1] if isinstance(vdim_param.bounds[1], int) else None
            if kdims and vdims is None:
                vdims = [c for c in data.columns if c not in kdims]
            elif vdims and kdims is None:
                kdims = [c for c in data.columns if c not in vdims][:ndim]
            elif kdims is None:
                kdims = list(data.columns[:ndim])
                if vdims is None:
                    vdims = [d for d in data.columns[ndim:((ndim+nvdim) if nvdim else None)]
                             if d not in kdims]
            elif kdims == [] and vdims is None:
                vdims = list(data.columns[:nvdim if nvdim else None])

            # Handle reset of index if kdims reference index by name
            for kd in kdims:
                if isinstance(kd, Dimension): kd = kd.name
                if kd in data.columns:
                    continue
                if any(kd == ('index' if name is None else name)
                       for name in data.index.names):
                    data = data.reset_index()
                    break
            if any(isinstance(d, (np.int64, int)) for d in kdims+vdims):
                raise DataError("pandas DataFrame column names used as dimensions "
                                "must be strings not integers.", cls)
        else:
            # Check if data is of non-numeric type
            # Then use defined data type
            kdims = kdims if kdims else kdim_param.default
            vdims = vdims if vdims else vdim_param.default
            columns = [d.name if isinstance(d, Dimension) else d
                       for d in kdims+vdims]

            if isinstance(data, dict) and all(c in data for c in columns):
                data = cyODict(((d, data[d]) for d in columns))
            elif isinstance(data, (list, dict)) and data in ([], {}):
                data = None
            elif (isinstance(data, dict) and not all(d in data for d in columns) and<commit_msg>Fixed bug handling dask dataframe indexes (#2335)<commit_after>from __future__ import absolute_import

from distutils.version import LooseVersion

try:
    import itertools.izip as zip
except ImportError:
    pass

import numpy as np
import pandas as pd

from .interface import Interface, DataError
from ..dimension import Dimension
from ..element import Element
from ..dimension import OrderedDict as cyODict
from ..ndmapping import NdMapping, item_check
from .. import util


class PandasInterface(Interface):

    types = (pd.DataFrame if pd else None,)

    datatype = 'dataframe'

    @classmethod
    def dimension_type(cls, columns, dim):
        name = columns.get_dimension(dim, strict=True).name
        idx = list(columns.data.columns).index(name)
        return columns.data.dtypes[idx].type

    @classmethod
    def init(cls, eltype, data, kdims, vdims):
        element_params = eltype.params()
        kdim_param = element_params['kdims']
        vdim_param = element_params['vdims']
        if util.is_series(data):
            data = data.to_frame()
        if util.is_dataframe(data):
            if eltype._auto_indexable_1d and len(data.columns) == 1:
                data = data.reset_index()
            if isinstance(kdim_param.bounds[1], int):
                ndim = min([kdim_param.bounds[1], len(kdim_param.default)])
            else:
                ndim = None
            nvdim = vdim_param.bounds[1] if isinstance(vdim_param.bounds[1], int) else None
            if kdims and vdims is None:
                vdims = [c for c in data.columns if c not in kdims]
            elif vdims and kdims is None:
                kdims = [c for c in data.columns if c not in vdims][:ndim]
            elif kdims is None:
                kdims = list(data.columns[:ndim])
                if vdims is None:
                    vdims = [d for d in data.columns[ndim:((ndim+nvdim) if nvdim else None)]
                             if d not in kdims]
            elif kdims == [] and vdims is None:
                vdims = list(data.columns[:nvdim if nvdim else None])

            # Handle reset of index if kdims reference index by name
            index_names = data.index.names if isinstance(data, pd.DataFrame) else [data.index.name]
            for kd in kdims:
                if isinstance(kd, Dimension): kd = kd.name
                if kd in data.columns:
                    continue
                if any(kd == ('index' if name is None else name)
                       for name in index_names):
                    data = data.reset_index()
                    break
            if any(isinstance(d, (np.int64, int)) for d in kdims+vdims):
                raise DataError("pandas DataFrame column names used as dimensions "
                                "must be strings not integers.", cls)
        else:
            # Check if data is of non-numeric type
            # Then use defined data type
            kdims = kdims if kdims else kdim_param.default
            vdims = vdims if vdims else vdim_param.default
            columns = [d.name if isinstance(d, Dimension) else d
                       for d in kdims+vdims]

            if isinstance(data, dict) and all(c in data for c in columns):
                data = cyODict(((d, data[d]) for d in columns))
            elif isinstance(data, (list, dict)) and data in ([], {}):
                data = None
            elif (isinstance(data, dict) and not all(d in data for d in columns) and
{'fraction_non_alphanumeric': 0.08263736263736264, 'fraction_numerical': 0.0029304029304029304, 'mean_word_length': 1.8680672268907563, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6861322', 'n_tokens_mistral': 1943, 'n_tokens_neox': 1931, 'n_words': 660}