<commit_before># -*- coding: utf-8 -*-
"""
Created on Sun Mar 12 13:16:47 2017

@author: Villanueva
"""


import sys
#import sys
import logging
# nice debug printing of settings
#import pprint
from typing import Dict, List, Tuple
from typing import Callable, TypeVar
from typing import Union, Sequence, Iterable, Mapping, Collection  # type: ignore
from functools import wraps
from enum import Enum
import warnings

from simetuc.util import ConfigWarning


## example on how to add a type to the typing module, in this case a OrderedDict
#import collections
#import typing
#class OrderedDictType(collections.OrderedDict, typing.MutableMapping[typing.KT, typing.VT],
#                      extra=collections.OrderedDict):
#    __slots__ = ()
#    def __new__(cls, *args, **kwds):
#        if typing._geqv(cls, OrderedDictType):
#            raise TypeError("Type OrderedDictType cannot be instantiated; "
#                            "use collections.OrderedDict() instead")
#        return typing._generic_new(collections.OrderedDict, cls, *args, **kwds)
#typing.OrderedDictType = OrderedDictType
#typing.OrderedDictType.__module__ = 'typing'


# type of innermost type, could be int, str, etc
T = TypeVar('T')
# the type of the settings value can be a simple type or a collection or types
ValType = Union[T, Collection[T]]

class Value():
    '''A value of a setting. The value has an specific type and optionally max and min values.
        If the setting is a list, the list can have max and min length.
        The type can be nested, eg: a list of lists of ints; use the typing module for that:
        type=typing.List[typing.List[int]].
        If the setting is a nested sequence, the max and min length
        can be lists of the length of each list.
        At any level of the type tree a Union of types is allowed; if present, the leftmost type
        in the Union will be parsed, if it doesn't succeed the next type will be parsed and so on.
        The innermost type must be concrete or a typing.Union of several concrete types.
        In this case the type of the settings will be checked agains in the values in the Union
        (in the same order) until one matches, or if none, an exception is raised.'''

    class Kind(Enum):
        mandatory = 1
        optional = 2
        exclusive = 3

    mandatory = Kind.mandatory
    optional = Kind.optional
    exclusive = Kind.exclusive

    # len_max/min are ints b/c the length of a Sequence is always an int.
    def __init__(self, name: str, val_type: ValType,
                 val_max: T = None, val_min: T = None,
                 kind: Kind = Kind.mandatory,
                 len_max: Union[int, List[int]] = None,
                 len_min: Union[int, List[int]] = None) -> None:
        '''Val type can be a nested type (List[int], List[List[int]])'''

        self.name = name
        # val_type it's either a simple type, an Iterable or a Union
        # Nesting is possible
        self.val_type = val_type

        self.val_max = val_max
        self.val_min = val_min

        self.kind = kind

        # convert to list
        self.len_max = [len_max] if not isinstance(len_max, Sequence) else len_max
        self.len_min = [len_min] if not isinstance(len_min, Sequence) else len_min

        self.no_logging = False

    def __repr__(self) -> str:
        '''Return a representation of Value'''

        return '{}({}, {})'.format(self.__class__.__name__, self.name,
                                   _get_clean_type_name(self.val_type))

    @staticmethod
    def _print_trace(value: ValType, val_type: ValType) -> None:  # pragma: no cover
        '''Print the current state of the tree parsing.'''
        type_name = _get_clean_type_name(val_type)
        print(value, type(value).__name__, type_name)

    def _check_val_max_min(self, value: T) -> None:
        '''Check that the value is within the max and min values.'''
        logger = logging.getLogger(__name__)

        # the type: ignore comments are there because it's possible that the user will
        # ask for an unorderable type but also give a max or min values.
        # That would fail, but that's the user's fault.
        if self.val_max is not None and value > self.val_max:  # type: ignore
            msg = ('Value(s) of {} ({}) cannot be '.format(self.name, value) +
                   'larger than {}.'.format(self.val_max))
            logger.error(msg)
            raise ValueError(msg)
        if self.val_min is not None and value < self.val_min:  # type: ignore
            msg = ('Value(s) of {} ({}) cannot be '.format(self.name, value) +
                   'smaller than {}.'.format(self.val_min))
            logger.error(msg)
            raise ValueError(msg)

    def _check_seq_len(self, seq: Collection, len_max: int, len_min: int) -> None:
        '''Checks that the sequence has the size given by self.len_max/min.'''
        logger = logging.getLogger(__name__)

        if len_max is not None and len(seq) > len_max:
            msg = ('Length of {} ({}) cannot be '.format(self.name, len(seq)) +
                   'larger than {}.'.format(len_max))
            logger.error(msg)
            raise ValueError(msg)
        if len_min is not None and len(seq) < len_min:
            msg = ('Length of {} ({}) cannot be '.format(self.name, len(seq)) +
                   'smaller than {}.'.format(len_min))
            logger.error(msg)<commit_msg>Collections not in py 3.5<commit_after># -*- coding: utf-8 -*-
"""
Created on Sun Mar 12 13:16:47 2017

@author: Villanueva
"""


import sys
#import sys
import logging
# nice debug printing of settings
#import pprint
from typing import Dict, List, Tuple
from typing import Callable, TypeVar
from typing import Union, Sequence, Iterable, Mapping, Sized
from functools import wraps
from enum import Enum
import warnings

from simetuc.util import ConfigWarning


## example on how to add a type to the typing module, in this case a OrderedDict
#import collections
#import typing
#class OrderedDictType(collections.OrderedDict, typing.MutableMapping[typing.KT, typing.VT],
#                      extra=collections.OrderedDict):
#    __slots__ = ()
#    def __new__(cls, *args, **kwds):
#        if typing._geqv(cls, OrderedDictType):
#            raise TypeError("Type OrderedDictType cannot be instantiated; "
#                            "use collections.OrderedDict() instead")
#        return typing._generic_new(collections.OrderedDict, cls, *args, **kwds)
#typing.OrderedDictType = OrderedDictType
#typing.OrderedDictType.__module__ = 'typing'


# type of innermost type, could be int, str, etc
T = TypeVar('T')
# the type of the settings value can be a simple type or a collection or types
ValType = Union[T, Iterable[T]]

class Value():
    '''A value of a setting. The value has an specific type and optionally max and min values.
        If the setting is a list, the list can have max and min length.
        The type can be nested, eg: a list of lists of ints; use the typing module for that:
        type=typing.List[typing.List[int]].
        If the setting is a nested sequence, the max and min length
        can be lists of the length of each list.
        At any level of the type tree a Union of types is allowed; if present, the leftmost type
        in the Union will be parsed, if it doesn't succeed the next type will be parsed and so on.
        The innermost type must be concrete or a typing.Union of several concrete types.
        In this case the type of the settings will be checked agains in the values in the Union
        (in the same order) until one matches, or if none, an exception is raised.'''

    class Kind(Enum):
        mandatory = 1
        optional = 2
        exclusive = 3

    mandatory = Kind.mandatory
    optional = Kind.optional
    exclusive = Kind.exclusive

    # len_max/min are ints b/c the length of a Sequence is always an int.
    def __init__(self, name: str, val_type: ValType,
                 val_max: T = None, val_min: T = None,
                 kind: Kind = Kind.mandatory,
                 len_max: Union[int, List[int]] = None,
                 len_min: Union[int, List[int]] = None) -> None:
        '''Val type can be a nested type (List[int], List[List[int]])'''

        self.name = name
        # val_type it's either a simple type, an Iterable or a Union
        # Nesting is possible
        self.val_type = val_type

        self.val_max = val_max
        self.val_min = val_min

        self.kind = kind

        # convert to list
        self.len_max = [len_max] if not isinstance(len_max, Sequence) else len_max
        self.len_min = [len_min] if not isinstance(len_min, Sequence) else len_min

        self.no_logging = False

    def __repr__(self) -> str:
        '''Return a representation of Value'''

        return '{}({}, {})'.format(self.__class__.__name__, self.name,
                                   _get_clean_type_name(self.val_type))

    @staticmethod
    def _print_trace(value: ValType, val_type: ValType) -> None:  # pragma: no cover
        '''Print the current state of the tree parsing.'''
        type_name = _get_clean_type_name(val_type)
        print(value, type(value).__name__, type_name)

    def _check_val_max_min(self, value: T) -> None:
        '''Check that the value is within the max and min values.'''
        logger = logging.getLogger(__name__)

        # the type: ignore comments are there because it's possible that the user will
        # ask for an unorderable type but also give a max or min values.
        # That would fail, but that's the user's fault.
        if self.val_max is not None and value > self.val_max:  # type: ignore
            msg = ('Value(s) of {} ({}) cannot be '.format(self.name, value) +
                   'larger than {}.'.format(self.val_max))
            logger.error(msg)
            raise ValueError(msg)
        if self.val_min is not None and value < self.val_min:  # type: ignore
            msg = ('Value(s) of {} ({}) cannot be '.format(self.name, value) +
                   'smaller than {}.'.format(self.val_min))
            logger.error(msg)
            raise ValueError(msg)

    def _check_seq_len(self, seq: Sized, len_max: int, len_min: int) -> None:
        '''Checks that the sequence has the size given by self.len_max/min.'''
        logger = logging.getLogger(__name__)

        if len_max is not None and len(seq) > len_max:
            msg = ('Length of {} ({}) cannot be '.format(self.name, len(seq)) +
                   'larger than {}.'.format(len_max))
            logger.error(msg)
            raise ValueError(msg)
        if len_min is not None and len(seq) < len_min:
            msg = ('Length of {} ({}) cannot be '.format(self.name, len(seq)) +
                   'smaller than {}.'.format(len_min))
            logger.error(msg)
{'fraction_non_alphanumeric': 0.09576036866359447, 'fraction_numerical': 0.0031336405529953917, 'mean_word_length': 2.4778846153846152, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 17, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4776823', 'n_tokens_mistral': 3188, 'n_tokens_neox': 3101, 'n_words': 1201}