<commit_before>import param
import numpy as np

from ..element import Element, NdElement
from .. import util


class Interface(param.Parameterized):

    interfaces = {}

    datatype = None

    gridded = False

    @classmethod
    def register(cls, interface):
        cls.interfaces[interface.datatype] = interface


    @classmethod
    def cast(cls, dataset, datatype=None, cast_type=None):
        """
        Given a list of Dataset objects, cast them to the specified
        datatype (by default the format matching the current interface)
        with the given cast_type (if specified).
        """
        if len({type(c) for c in dataset}) > 1 and cast_type is None:
            raise Exception("Please supply the common cast type")

        if datatype is None:
           datatype = cls.datatype

        unchanged = all({c.interface==cls for c in dataset})
        if unchanged and cast_type is None:
            return dataset
        elif unchanged:
            return [cast_type(co, **dict(util.get_param_values(co)))
                    for co in dataset]

        return [co.clone(co.columns(), datatype=[datatype], new_type=cast_type)
                for co in dataset]


    @classmethod
    def initialize(cls, eltype, data, kdims, vdims, datatype=None):
        # Process params and dimensions
        if isinstance(data, Element):
            pvals = util.get_param_values(data)
            kdims = pvals.get('kdims') if kdims is None else kdims
            vdims = pvals.get('vdims') if vdims is None else vdims

        # Process Element data
        if isinstance(data, NdElement):
            kdims = [kdim for kdim in kdims if kdim != 'Index']
        elif hasattr(data, 'interface') and issubclass(data.interface, Interface):
            data = data.data
        elif isinstance(data, Element):
            data = tuple(data.dimension_values(d) for d in kdims+vdims)
        elif isinstance(data, util.generator_types):
            data = list(data)

        # Set interface priority order
        if datatype is None:
            datatype = eltype.datatype
        prioritized = [cls.interfaces[p] for p in datatype
                       if p in cls.interfaces]

        head = [intfc for intfc in prioritized if type(data) in intfc.types]
        if head:
            # Prioritize interfaces which have matching types
            prioritized = head + [el for el in prioritized if el != head[0]]

        # Iterate over interfaces until one can interpret the input
        for interface in prioritized:
            try:
                (data, dims, extra_kws) = interface.init(eltype, data, kdims, vdims)
                break
            except:
                pass
        else:
            raise ValueError("None of the available storage backends "
                             "were able to support the supplied data format.")

        return data, interface, dims, extra_kws


    @classmethod
    def validate(cls, dataset):
        not_found = [d for d in dataset.dimensions(label='name')
                     if d not in dataset.data]
        if not_found:
            raise ValueError("Supplied data does not contain specified "
                             "dimensions, the following dimensions were "<commit_msg>Improved casting between incompatible datatypes<commit_after>import param
import numpy as np

from ..element import Element, NdElement
from .. import util


class Interface(param.Parameterized):

    interfaces = {}

    datatype = None

    gridded = False

    @classmethod
    def register(cls, interface):
        cls.interfaces[interface.datatype] = interface


    @classmethod
    def cast(cls, dataset, datatype=None, cast_type=None):
        """
        Given a list of Dataset objects, cast them to the specified
        datatype (by default the format matching the current interface)
        with the given cast_type (if specified).
        """
        if len({type(c) for c in dataset}) > 1 and cast_type is None:
            raise Exception("Please supply the common cast type")

        if datatype is None:
           datatype = cls.datatype

        unchanged = all({c.interface==cls for c in dataset})
        if unchanged and cast_type is None:
            return dataset
        elif unchanged:
            return [cast_type(co, **dict(util.get_param_values(co)))
                    for co in dataset]

        return [co.clone(co.columns(), datatype=[datatype], new_type=cast_type)
                for co in dataset]


    @classmethod
    def initialize(cls, eltype, data, kdims, vdims, datatype=None):
        # Process params and dimensions
        if isinstance(data, Element):
            pvals = util.get_param_values(data)
            kdims = pvals.get('kdims') if kdims is None else kdims
            vdims = pvals.get('vdims') if vdims is None else vdims

        if datatype is None:
            datatype = eltype.datatype

        # Process Element data
        if isinstance(data, NdElement):
            kdims = [kdim for kdim in kdims if kdim != 'Index']
        elif (hasattr(data, 'interface') and issubclass(data.interface, Interface)):
            if data.interface.datatype in datatype:
                data = data.data
            elif data.interface.gridded:
                gridded = {data.dimension_values(kd.name, expanded=False) for kd in self.kdims}
                for vd in data.vdims:
                    gridded[vd.name] = data.dimension_values(vd, flat=False)
                data = gridded
            else:
                data = data.columns()
        elif isinstance(data, Element):
            data = tuple(data.dimension_values(d) for d in kdims+vdims)
        elif isinstance(data, util.generator_types):
            data = list(data)

        # Set interface priority order
        prioritized = [cls.interfaces[p] for p in datatype
                       if p in cls.interfaces]
        head = [intfc for intfc in prioritized if type(data) in intfc.types]
        if head:
            # Prioritize interfaces which have matching types
            prioritized = head + [el for el in prioritized if el != head[0]]

        # Iterate over interfaces until one can interpret the input
        for interface in prioritized:
            try:
                (data, dims, extra_kws) = interface.init(eltype, data, kdims, vdims)
                break
            except:
                pass
        else:
            raise ValueError("None of the available storage backends "
                             "were able to support the supplied data format.")

        return data, interface, dims, extra_kws


    @classmethod
    def validate(cls, dataset):
        not_found = [d for d in dataset.dimensions(label='name')
                     if d not in dataset.data]
        if not_found:
            raise ValueError("Supplied data does not contain specified "
                             "dimensions, the following dimensions were "
{'fraction_non_alphanumeric': 0.07643587159925147, 'fraction_numerical': 0.000575788109975529, 'mean_word_length': 1.9987052222701769, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6322619', 'n_tokens_mistral': 1864, 'n_tokens_neox': 1824, 'n_words': 641}