<commit_before>from __future__ import absolute_import
from .utils import from_iso8601, none_count, deref
import datetime
import functools
import six
import base64
import json


class Byte(object):
    """ for string type, byte format
    """

    def __init__(self, _, v):
        """ constructor

        :param str v: only accept six.string_types
        """
        if isinstance(v, six.string_types):
            self.v = v
        else:
            raise ValueError('Unsupported type for Byte: ' + str(type(v)))

    def __str__(self):
        return self.v

    def to_json(self):
        """ according to https://github.com/wordnik/swagger-spec/issues/50,
        we should exchange 'byte' type via base64 encoding.
        """
        return base64.urlsafe_b64encode(self.v)


class Time(object):
    """ Base of Datetime & Date
    """

    def __str__(self):
        return str(self.to_json())

    def to_json(self):
        # according to
        #   https://github.com/wordnik/swagger-spec/issues/95
        return self.v.isoformat()


class Date(Time):
    """ for string type, date format
    """

    def __init__(self, _, v):
        """ constructor

        :param v: things used to constrcut date
        :type v: timestamp in float, datetime.date object, or ISO-8601 in str
        """
        self.v = None
        if isinstance(v, float):
            self.v = datetime.date.fromtimestamp(v)
        elif isinstance(v, datetime.date):
            self.v = v
        elif isinstance(v, six.string_types):
            self.v = from_iso8601(v).date()
        else:
            raise ValueError('Unrecognized type for Date: ' + str(type(v)))


class Datetime(Time):
    """ for string type, datetime format
    """

    def __init__(self, _, v):
        """ constructor

        :param v: things used to constrcut date
        :type v: timestamp in float, datetime.datetime object, or ISO-8601 in str
        """
        self.v = None
        if isinstance(v, float):
            self.v = datetime.datetime.fromtimestamp(v)
        elif isinstance(v, datetime.datetime):
            self.v = v
        elif isinstance(v, six.string_types):
            self.v = from_iso8601(v)
        else:
            raise ValueError('Unrecognized type for Datetime: ' + str(type(v)))


class Array(list):
    """ for array type, or parameter when allowMultiple=True
    """

    def __init__(self):
        """ v: list or string_types
        """
        super(Array, self).__init__()
        self.__collection_format = 'csv'

    def apply_with(self, obj, val, _):
        """
        """
        if isinstance(val, six.string_types):
            val = json.loads(val)

        val = set(val) if obj.uniqueItems else val

        if obj.items and len(val):
            self.extend(map(functools.partial(prim_factory, obj.items), val))
            val = []

        # init array as list
        if obj.minItems and len(self) < obj.minItems:
            raise ValueError('Array should be more than {0}, not {1}'.format(o.minItems, len(self)))
        if obj.maxItems and len(self) > obj.maxItems:
            raise ValueError('Array should be less than {0}, not {1}'.format(o.maxItems, len(self)))

        self.__collection_format = getattr(obj, 'collectionFormat', 'csv')
        return val

    def __str__(self):
        """ array primitives should be for 'path', 'header', 'query'.
        Therefore, this kind of convertion is reasonable.

        :return: the converted string
        :rtype: str
        """
        def _conv(p):
            s = ''
            for v in self:
                s = ''.join([s, p if s else '', str(v)])
            return s
    
        if self.__collection_format == 'csv':
            return _conv(',')
        elif self.__collection_format == 'ssv':
            return _conv(' ')
        elif self.__collection_format == 'tsv':
            return _conv('\t')
        elif self.__collection_format == 'pipes':
            return _conv('|')
        else:
            raise ValueError('Unsupported collection format when converting to str: {0}'.format(self.__collection_format))

    def to_url(self):
        """ special function for handling 'multi',
        refer to Swagger 2.0, Parameter Object, collectionFormat
        """
        if self.__collection_format == 'multi':
            return [str(s) for s in self]
        else:
            return [str(self)]


class Model(dict):
    """ for complex type: models
    """

    # access dict like object
    __getattr__ = dict.__getitem__
    __setattr__ = dict.__setitem__

    __name_key__ = '!!_name____!!'

    def __init__(self, o):
        """ constructor
        """
        super(Model, self).__init__()

    def apply_with(self, obj, val, ctx):
        """ recursivly apply Schema object

        :param obj.Model obj: model object to instruct how to create this model
        :param val: things used to construct this model
        :type val: dict of json string in str or byte
        """
        if isinstance(val, six.string_types):
            val = json.loads(val)
        elif isinstance(val, six.binary_type):
            # TODO: encoding problem...
            val = json.loads(val.decode('utf-8'))

        # init model as dict
        for k, v in six.iteritems(obj.properties):
            to_update = val.pop(k, None)

            # update discriminator with model's name
            if obj.discriminator and obj.discriminator == k:
                to_update = ctx['name']

            # check require properties of a Model
            if to_update == None:
                if obj.required and k in obj.required:
                    raise ValueError('Model:[' + str(obj.name) + '], require:[' + str(k) + ']')

            self[k] = prim_factory(v, to_update)

        return val

    def __eq__(self, other):
        """ equality operater, 
        will skip checking when both value are None or no attribute.

        :param other: another model
        :type other: primitives.Model or dict
        """
        if other == None:
            return False
<commit_msg>should be get instead of pop<commit_after>from __future__ import absolute_import
from .utils import from_iso8601, none_count, deref
import datetime
import functools
import six
import base64
import json


class Byte(object):
    """ for string type, byte format
    """

    def __init__(self, _, v):
        """ constructor

        :param str v: only accept six.string_types
        """
        if isinstance(v, six.string_types):
            self.v = v
        else:
            raise ValueError('Unsupported type for Byte: ' + str(type(v)))

    def __str__(self):
        return self.v

    def to_json(self):
        """ according to https://github.com/wordnik/swagger-spec/issues/50,
        we should exchange 'byte' type via base64 encoding.
        """
        return base64.urlsafe_b64encode(self.v)


class Time(object):
    """ Base of Datetime & Date
    """

    def __str__(self):
        return str(self.to_json())

    def to_json(self):
        # according to
        #   https://github.com/wordnik/swagger-spec/issues/95
        return self.v.isoformat()


class Date(Time):
    """ for string type, date format
    """

    def __init__(self, _, v):
        """ constructor

        :param v: things used to constrcut date
        :type v: timestamp in float, datetime.date object, or ISO-8601 in str
        """
        self.v = None
        if isinstance(v, float):
            self.v = datetime.date.fromtimestamp(v)
        elif isinstance(v, datetime.date):
            self.v = v
        elif isinstance(v, six.string_types):
            self.v = from_iso8601(v).date()
        else:
            raise ValueError('Unrecognized type for Date: ' + str(type(v)))


class Datetime(Time):
    """ for string type, datetime format
    """

    def __init__(self, _, v):
        """ constructor

        :param v: things used to constrcut date
        :type v: timestamp in float, datetime.datetime object, or ISO-8601 in str
        """
        self.v = None
        if isinstance(v, float):
            self.v = datetime.datetime.fromtimestamp(v)
        elif isinstance(v, datetime.datetime):
            self.v = v
        elif isinstance(v, six.string_types):
            self.v = from_iso8601(v)
        else:
            raise ValueError('Unrecognized type for Datetime: ' + str(type(v)))


class Array(list):
    """ for array type, or parameter when allowMultiple=True
    """

    def __init__(self):
        """ v: list or string_types
        """
        super(Array, self).__init__()
        self.__collection_format = 'csv'

    def apply_with(self, obj, val, _):
        """
        """
        if isinstance(val, six.string_types):
            val = json.loads(val)

        val = set(val) if obj.uniqueItems else val

        if obj.items and len(val):
            self.extend(map(functools.partial(prim_factory, obj.items), val))
            val = []

        # init array as list
        if obj.minItems and len(self) < obj.minItems:
            raise ValueError('Array should be more than {0}, not {1}'.format(o.minItems, len(self)))
        if obj.maxItems and len(self) > obj.maxItems:
            raise ValueError('Array should be less than {0}, not {1}'.format(o.maxItems, len(self)))

        self.__collection_format = getattr(obj, 'collectionFormat', 'csv')
        return val

    def __str__(self):
        """ array primitives should be for 'path', 'header', 'query'.
        Therefore, this kind of convertion is reasonable.

        :return: the converted string
        :rtype: str
        """
        def _conv(p):
            s = ''
            for v in self:
                s = ''.join([s, p if s else '', str(v)])
            return s
    
        if self.__collection_format == 'csv':
            return _conv(',')
        elif self.__collection_format == 'ssv':
            return _conv(' ')
        elif self.__collection_format == 'tsv':
            return _conv('\t')
        elif self.__collection_format == 'pipes':
            return _conv('|')
        else:
            raise ValueError('Unsupported collection format when converting to str: {0}'.format(self.__collection_format))

    def to_url(self):
        """ special function for handling 'multi',
        refer to Swagger 2.0, Parameter Object, collectionFormat
        """
        if self.__collection_format == 'multi':
            return [str(s) for s in self]
        else:
            return [str(self)]


class Model(dict):
    """ for complex type: models
    """

    # access dict like object
    __getattr__ = dict.__getitem__
    __setattr__ = dict.__setitem__

    __name_key__ = '!!_name____!!'

    def __init__(self, o):
        """ constructor
        """
        super(Model, self).__init__()

    def apply_with(self, obj, val, ctx):
        """ recursivly apply Schema object

        :param obj.Model obj: model object to instruct how to create this model
        :param val: things used to construct this model
        :type val: dict of json string in str or byte
        """
        if isinstance(val, six.string_types):
            val = json.loads(val)
        elif isinstance(val, six.binary_type):
            # TODO: encoding problem...
            val = json.loads(val.decode('utf-8'))

        # init model as dict
        for k, v in six.iteritems(obj.properties):
            to_update = val.get(k, None)

            # update discriminator with model's name
            if obj.discriminator and obj.discriminator == k:
                to_update = ctx['name']

            # check require properties of a Model
            if to_update == None:
                if obj.required and k in obj.required:
                    raise ValueError('Model:[' + str(obj.name) + '], require:[' + str(k) + ']')

            self[k] = prim_factory(v, to_update)

        return val

    def __eq__(self, other):
        """ equality operater, 
        will skip checking when both value are None or no attribute.

        :param other: another model
        :type other: primitives.Model or dict
        """
        if other == None:
            return False

{'fraction_non_alphanumeric': 0.11146679799720922, 'fraction_numerical': 0.006566527127965197, 'mean_word_length': 2.0024642681123708, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 4, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7624137', 'n_tokens_mistral': 3496, 'n_tokens_neox': 3395, 'n_words': 1047}