<commit_before>##
# Copyright (c) 2008-2010 Sprymix Inc.
# All rights reserved.
#
# See LICENSE for details.
##


import uuid


from semantix.caos import types as caos_types


def check_type(variable, type):
    if not isinstance(type, str):
        raise Exception('check_type: type parameter must be string')

    if variable is None:
        return True

    if type == 'str':
        return isinstance(variable, str)

    if type == 'int':
        return isinstance(variable, int)

    if type == 'float':
        return isinstance(variable, float)

    if type == 'bool':
        return isinstance(variable, bool)

    if type == 'list':
        return isinstance(variable, list)

    if type == 'uuid':
        return isinstance(variable, uuid.UUID)

    if type == 'none':
        return variable is None

    raise Exception('check_type: checking on unknown type: %s' % type)


class DatasourceError(Exception):
    pass


class Datasource(object):
    @classmethod
    def prepare_class(cls, context, descriptor):
        cls.descriptor = descriptor

    def __init__(self):
        self.params = self.descriptor.get('params', None)

    def describe_output(self):
        raise NotImplementedError

    def check_type(self, name, value, type):
        if check_type(value, type):
            return value
        else:
            raise ValueError('invalid parameter type')

    def _filter_params(self, params, filters=None):
        if self.params is None:
            return {}

        filtered = {}

        for name, config in self.params.items():
            value = None

            if name in params:
                value = params[name]

                try:
                    value = self.check_type(name, value, config['type'])
                except ValueError as e:
                    raise DatasourceError('datatype check failed, param: @name=%s, @value=%s, expected type: %s' %
                                          (name, value, config['type'])) from e
            else:
                if 'default' in config:
                    value = self.coerce_default_value(name, config['default'], config['type'])
                else:
                    raise DatasourceError('expected required param: @name=%s' % name)

            filtered[name] = value

        if filters:
            extra_filters = filters.copy()

            for name, value in filters.items():
                if name not in self.params:
                    filtered['__filter%s' % name] = value
                else:
                    filtered[name] = self.check_type(name, value, self.params[name]['type'])
                    del extra_filters[name]
        else:
            extra_filters = None

        return filtered, extra_filters

    def fetch(self, *, _filters=None, _sort=None, **params):
        raise NotImplementedError

    def coerce_default_value(self, name, value, type):
        return value


class CaosDatasource(Datasource):
    def __init__(self, session):
        super().__init__()
        self.session = session

        arg_types = {}

        for arg_name, arg in self.params.items():
            if isinstance(arg['type'], (list, tuple)):
                arg_type = self.session.proto_schema.get(arg['type'][0])
                arg_type = (list, arg_type)
            else:
                arg_type = self.session.proto_schema.get(arg['type'])
            arg_types[arg_name] = arg_type<commit_msg>caos.datasources: Fix Python datasources regression<commit_after>##
# Copyright (c) 2008-2012 Sprymix Inc.
# All rights reserved.
#
# See LICENSE for details.
##


import uuid


from semantix.caos import types as caos_types


def check_type(variable, type):
    if not isinstance(type, str):
        raise Exception('check_type: type parameter must be string')

    if variable is None:
        return True

    if type == 'str':
        return isinstance(variable, str)

    if type == 'int':
        return isinstance(variable, int)

    if type == 'float':
        return isinstance(variable, float)

    if type == 'bool':
        return isinstance(variable, bool)

    if type == 'list':
        return isinstance(variable, list)

    if type == 'uuid':
        return isinstance(variable, uuid.UUID)

    if type == 'none':
        return variable is None

    raise Exception('check_type: checking on unknown type: %s' % type)


class DatasourceError(Exception):
    pass


class Datasource(object):
    @classmethod
    def prepare_class(cls, context, descriptor):
        cls.descriptor = descriptor

    def __init__(self):
        self.params = self.descriptor.get('params', None)

    def describe_output(self):
        raise NotImplementedError

    def check_type(self, name, value, type):
        if check_type(value, type):
            return value
        else:
            raise ValueError('invalid parameter type')

    def _filter_params(self, params, filters=None):
        if self.params is None:
            return {}

        filtered = {}

        for name, config in self.params.items():
            value = None

            if name in params:
                value = params[name]

                try:
                    value = self.check_type(name, value, config['type'])
                except ValueError as e:
                    raise DatasourceError('datatype check failed, param: @name=%s, @value=%s, expected type: %s' %
                                          (name, value, config['type'])) from e
            else:
                if 'default' in config:
                    value = self.coerce_default_value(name, config['default'], config['type'])
                else:
                    raise DatasourceError('expected required param: @name=%s' % name)

            filtered[name] = value

        if filters:
            extra_filters = filters.copy()

            for name, value in filters.items():
                if name not in self.params:
                    filtered['__filter%s' % name] = value
                else:
                    filtered[name] = self.check_type(name, value, self.params[name]['type'])
                    del extra_filters[name]
        else:
            extra_filters = None

        return filtered, extra_filters

    def fetch(self, *, _filters=None, _sort=None, **params):
        raise NotImplementedError

    def coerce_default_value(self, name, value, type):
        return value


class CaosDatasource(Datasource):
    def __init__(self, session):
        super().__init__()
        self.session = session

        self._init_arg_types()

    def _init_arg_types(self):
        arg_types = {}

        for arg_name, arg in self.params.items():
            if isinstance(arg['type'], (list, tuple)):
                arg_type = self.session.proto_schema.get(arg['type'][0])
                arg_type = (list, arg_type)
            else:
                arg_type = self.session.proto_schema.get(arg['type'])
            arg_types[arg_name] = arg_type
{'fraction_non_alphanumeric': 0.10213742648113613, 'fraction_numerical': 0.002582125950365801, 'mean_word_length': 1.9884269181311616, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '106208', 'n_tokens_mistral': 1925, 'n_tokens_neox': 1818, 'n_words': 536}