<commit_before>from __future__ import absolute_import
from __future__ import division
from __future__ import print_function
from __future__ import unicode_literals

from collections import namedtuple
import json
import numpy as np
import six
import tempfile

from .base import Property


FileProp = namedtuple('FileProp', ['file', 'dtype'])


class Array(Property):
    """Array Property with specified shape and type.

    Currently only float and int arrays are supported.
    """

    _shape = ('*',)
    _dtype = (float, int)

    _sphinx_prefix = 'properties'

    @property
    def doc(self):
        if getattr(self, '_doc', None) is None:
            self._doc = '{doc}, shape: {shp}, type: {dtype}'.format(
                doc=self._base_doc,
                shp='(' + ', '.join(['\*' if s == '*' else str(s)
                                     for s in self.shape]) + ')',
                dtype=self.dtype
            )
        return self._doc

    def serialize(self, data):
        """Convert the array data to a serialized binary format"""
        if isinstance(data.flatten()[0], np.floating):
            use_dtype = '<f4'
            assert np.allclose(data.astype(use_dtype), data, equal_nan=True), \
                'Converting the type should not screw things up.'
        elif isinstance(data.flatten()[0], np.integer):
            use_dtype = '<i4'
            assert (data.astype(use_dtype) == data).all(), \
                'Converting the type should not screw things up.'
        else:
            raise Exception('Must be a float or an int: {}'.format(data.dtype))

        data_file = tempfile.NamedTemporaryFile('rb+', suffix='.dat')
        data_file.write(data.astype(use_dtype).tobytes())
        data_file.seek(0)
        return FileProp(data_file, use_dtype)

    @property
    def shape(self):
        return self._shape

    @shape.setter
    def shape(self, value):
        if not isinstance(value, tuple):
            raise TypeError("{}: Invalid shape - must be a tuple "
                            "(e.g. ('*',3) for an array of length-3 "
                            "arrays)".format(value, self.name))
        for s in value:
            if s != '*' and not isinstance(s, six.integer_types):
                raise TypeError("{}: Invalid shape - values "
                                "must be '*' or int".format(
                                    value, self.name
                                ))
        self._shape = value

    @property
    def dtype(self):
        return self._dtype

    @dtype.setter
    def dtype(self, value):
        if not isinstance(value, (list, tuple)):
            value = (value,)
        if (float not in value and
                len(set(value).intersection(six.integer_types)) == 0):
            raise TypeError("{}: Invalid dtype - must be int "
                            "and/or float".format(value, self.name))
        self._dtype = value


    @property
    def _schema_function(self):
        if getattr(self, '__schema_function', None) is not None:
            return self.__schema_function

        def test_function(proposed):
            err_str = self.name
            if (proposed.dtype.kind == 'i' and
                    len(set(self.dtype).intersection(six.integer_types)) == 0):
                raise ValueError('{}: Array type must be int'.format(err_str))
            if proposed.dtype.kind == 'f' and float not in self.dtype:
                raise ValueError('{}: Array type must be '
                                 'float'.format(err_str))
            if len(self.shape) != proposed.ndim:
                raise ValueError(
                    '{}: Array must have {:d} dimensions ''(shape: {})'.format(
                        err_str, len(self.shape), self.shape
                    ))
            for i, s in enumerate(self.shape):
                if s != '*' and proposed.shape[i] != s:
                    raise ValueError('{}: Array dimension {:d} must be '
                                     'length {:d}'.format(err_str, i, s))

        self.__schema_function = test_function
        return test_function

    def validator(self, instance, proposed):
        if not isinstance(proposed, (list, np.ndarray)):
            raise ValueError('{} must be a list or numpy array'.format(
                self.name
            ))
        proposed = np.array(proposed)
        self._schema_function(proposed)
        return proposed

    def from_JSON(self, value):
        return json.loads(value)<commit_msg>Simplify validation/schema function<commit_after>from __future__ import absolute_import
from __future__ import division
from __future__ import print_function
from __future__ import unicode_literals

from collections import namedtuple
import json
import numpy as np
import six
import tempfile

from .base import Property


FileProp = namedtuple('FileProp', ['file', 'dtype'])


class Array(Property):
    """Array Property with specified shape and type.

    Currently only float and int arrays are supported.
    """

    _shape = ('*',)
    _dtype = (float, int)

    _sphinx_prefix = 'properties'

    @property
    def doc(self):
        if getattr(self, '_doc', None) is None:
            self._doc = '{doc}, shape: {shp}, type: {dtype}'.format(
                doc=self._base_doc,
                shp='(' + ', '.join(['\*' if s == '*' else str(s)
                                     for s in self.shape]) + ')',
                dtype=self.dtype
            )
        return self._doc

    def serialize(self, data):
        """Convert the array data to a serialized binary format"""
        if isinstance(data.flatten()[0], np.floating):
            use_dtype = '<f4'
            assert np.allclose(data.astype(use_dtype), data, equal_nan=True), \
                'Converting the type should not screw things up.'
        elif isinstance(data.flatten()[0], np.integer):
            use_dtype = '<i4'
            assert (data.astype(use_dtype) == data).all(), \
                'Converting the type should not screw things up.'
        else:
            raise Exception('Must be a float or an int: {}'.format(data.dtype))

        data_file = tempfile.NamedTemporaryFile('rb+', suffix='.dat')
        data_file.write(data.astype(use_dtype).tobytes())
        data_file.seek(0)
        return FileProp(data_file, use_dtype)

    @property
    def shape(self):
        return self._shape

    @shape.setter
    def shape(self, value):
        if not isinstance(value, tuple):
            raise TypeError("{}: Invalid shape - must be a tuple "
                            "(e.g. ('*',3) for an array of length-3 "
                            "arrays)".format(value, self.name))
        for s in value:
            if s != '*' and not isinstance(s, six.integer_types):
                raise TypeError("{}: Invalid shape - values "
                                "must be '*' or int".format(
                                    value, self.name
                                ))
        self._shape = value

    @property
    def dtype(self):
        return self._dtype

    @dtype.setter
    def dtype(self, value):
        if not isinstance(value, (list, tuple)):
            value = (value,)
        if (float not in value and
                len(set(value).intersection(six.integer_types)) == 0):
            raise TypeError("{}: Invalid dtype - must be int "
                            "and/or float".format(value, self.name))
        self._dtype = value

    def validator(self, instance, proposed):
        if not isinstance(proposed, (list, np.ndarray)):
            raise ValueError('{} must be a list or numpy array'.format(
                self.name
            ))
        proposed = np.array(proposed)
        if (proposed.dtype.kind == 'i' and
                len(set(self.dtype).intersection(six.integer_types)) == 0):
            raise ValueError('{}: Array type must be int'.format(self.name))
        if proposed.dtype.kind == 'f' and float not in self.dtype:
            raise ValueError('{}: Array type must be '
                             'float'.format(self.name))
        if len(self.shape) != proposed.ndim:
            raise ValueError(
                '{}: Array must have {:d} dimensions ''(shape: {})'.format(
                    self.name, len(self.shape), self.shape
                ))
        for i, s in enumerate(self.shape):
            if s != '*' and proposed.shape[i] != s:
                raise ValueError('{}: Array dimension {:d} must be '
                                 'length {:d}'.format(self.name, i, s))
        return proposed

    def from_JSON(self, value):
        return json.loads(value)
{'fraction_non_alphanumeric': 0.11671669360424844, 'fraction_numerical': 0.0020780420226275687, 'mean_word_length': 1.8062844185293165, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 9, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1313773', 'n_tokens_mistral': 2397, 'n_tokens_neox': 2344, 'n_words': 690}