<commit_before>from strictyaml.exceptions import YAMLSerializationError
from strictyaml.validators import Validator
from strictyaml.representation import YAML
from strictyaml import constants
from strictyaml import utils
from datetime import datetime
import dateutil.parser
import decimal
import sys
import re
from ruamel.yaml.scalarstring import PreservedScalarString


if sys.version_info[0] == 3:
    unicode = str


class ScalarValidator(Validator):
    @property
    def rule_description(self):
        return "a {0}".format(self.__class__.__name__.lower())

    def __call__(self, chunk):
        chunk.expect_scalar(self.rule_description)
        return YAML(chunk, validator=self)

    def validate(self, chunk):
        return self.validate_scalar(chunk)

    def should_be_string(self, data, message):
        if not utils.is_string(data):
            raise YAMLSerializationError(
                "{0} got '{1}' of type {2}.".format(message, data, type(data).__name__)
            )

    def validate_scalar(self, chunk):
        raise NotImplementedError("validate_scalar(self, chunk) must be implemented")


class Enum(ScalarValidator):
    def __init__(self, restricted_to, item_validator=None):
        self._item_validator = Str() if item_validator is None else item_validator
        assert isinstance(
            self._item_validator, ScalarValidator
        ), "item validator must be scalar too"
        self._restricted_to = restricted_to

    def validate_scalar(self, chunk):
        val = self._item_validator(chunk)
        val._validator = self
        if val.scalar not in self._restricted_to:
            chunk.expecting_but_found(
                "when expecting one of: {0}".format(", ".join(self._restricted_to))
            )
        else:
            return val

    def to_yaml(self, data):
        if data not in self._restricted_to:
            raise YAMLSerializationError(
                "Got '{0}' when  expecting one of: {1}".format(
                    data, ", ".join(self._restricted_to)
                )
            )
        return self._item_validator.to_yaml(data)

    def __repr__(self):
        # TODO : item_validator
        return u"Enum({0})".format(repr(self._restricted_to))


class CommaSeparated(ScalarValidator):
    def __init__(self, item_validator):
        self._item_validator = item_validator
        assert isinstance(
            self._item_validator, ScalarValidator
        ), "item validator must be scalar too"

    def validate_scalar(self, chunk):
        if chunk.contents == "":
            return []
        return [
            self._item_validator.validate_scalar(
                chunk.textslice(positions[0], positions[1])
            )
            for positions in utils.comma_separated_positions(chunk.contents)
        ]

    def to_yaml(self, data):
        if isinstance(data, list):
            return ", ".join([self._item_validator.to_yaml(item) for item in data])
        elif utils.is_string(data):
            for item in data.split(","):
                self._item_validator.to_yaml(item)
            return data
        else:
            raise YAMLSerializationError(
                "expected string or list, got '{}' of type '{}'".format(
                    data, type(data).__name__
                )
            )

    def __repr__(self):
        return "CommaSeparated({0})".format(self._item_validator)


class Regex(ScalarValidator):
    def __init__(self, regular_expression):
        """
        Give regular expression, e.g. u'[0-9]'
        """
        self._regex = regular_expression
        self._matching_message = "when expecting string matching {0}".format(
            self._regex
        )

    def validate_scalar(self, chunk):
        if re.compile(self._regex).match(chunk.contents) is None:
            chunk.expecting_but_found(
                self._matching_message, "found non-matching string"
            )
        return chunk.contents

    def to_yaml(self, data):
        self.should_be_string(data, self._matching_message)
        if re.compile(self._regex).match(data) is None:
            raise YAMLSerializationError(
                "{} found '{}'".format(self._matching_message, data)
            )
        return data


class Email(Regex):
    def __init__(self):
        self._regex = constants.REGEXES["email"]
        self._matching_message = "when expecting an email address"


class Url(Regex):
    def __init__(self):
        self._regex = constants.REGEXES["url"]
        self._matching_message = "when expecting a url"


class Str(ScalarValidator):
    def validate_scalar(self, chunk):
        return chunk.contents

    def to_yaml(self, data):
        if not utils.is_string(data):
            raise YAMLSerializationError("'{}' is not a string".format(data))
        s = str(data)
        if u"\n" in s:
            s = PreservedScalarString(s)
        return s


class Int(ScalarValidator):
    def validate_scalar(self, chunk):
        val = chunk.contents
        if not utils.is_integer(val):<commit_msg>BUGFIX : Fixed python 2 bug introduced when fixing #72.<commit_after>from strictyaml.exceptions import YAMLSerializationError
from strictyaml.validators import Validator
from strictyaml.representation import YAML
from strictyaml import constants
from strictyaml import utils
from datetime import datetime
import dateutil.parser
import decimal
import sys
import re
from ruamel.yaml.scalarstring import PreservedScalarString


if sys.version_info[0] == 3:
    unicode = str


class ScalarValidator(Validator):
    @property
    def rule_description(self):
        return "a {0}".format(self.__class__.__name__.lower())

    def __call__(self, chunk):
        chunk.expect_scalar(self.rule_description)
        return YAML(chunk, validator=self)

    def validate(self, chunk):
        return self.validate_scalar(chunk)

    def should_be_string(self, data, message):
        if not utils.is_string(data):
            raise YAMLSerializationError(
                "{0} got '{1}' of type {2}.".format(message, data, type(data).__name__)
            )

    def validate_scalar(self, chunk):
        raise NotImplementedError("validate_scalar(self, chunk) must be implemented")


class Enum(ScalarValidator):
    def __init__(self, restricted_to, item_validator=None):
        self._item_validator = Str() if item_validator is None else item_validator
        assert isinstance(
            self._item_validator, ScalarValidator
        ), "item validator must be scalar too"
        self._restricted_to = restricted_to

    def validate_scalar(self, chunk):
        val = self._item_validator(chunk)
        val._validator = self
        if val.scalar not in self._restricted_to:
            chunk.expecting_but_found(
                "when expecting one of: {0}".format(", ".join(self._restricted_to))
            )
        else:
            return val

    def to_yaml(self, data):
        if data not in self._restricted_to:
            raise YAMLSerializationError(
                "Got '{0}' when  expecting one of: {1}".format(
                    data, ", ".join(self._restricted_to)
                )
            )
        return self._item_validator.to_yaml(data)

    def __repr__(self):
        # TODO : item_validator
        return u"Enum({0})".format(repr(self._restricted_to))


class CommaSeparated(ScalarValidator):
    def __init__(self, item_validator):
        self._item_validator = item_validator
        assert isinstance(
            self._item_validator, ScalarValidator
        ), "item validator must be scalar too"

    def validate_scalar(self, chunk):
        if chunk.contents == "":
            return []
        return [
            self._item_validator.validate_scalar(
                chunk.textslice(positions[0], positions[1])
            )
            for positions in utils.comma_separated_positions(chunk.contents)
        ]

    def to_yaml(self, data):
        if isinstance(data, list):
            return ", ".join([self._item_validator.to_yaml(item) for item in data])
        elif utils.is_string(data):
            for item in data.split(","):
                self._item_validator.to_yaml(item)
            return data
        else:
            raise YAMLSerializationError(
                "expected string or list, got '{}' of type '{}'".format(
                    data, type(data).__name__
                )
            )

    def __repr__(self):
        return "CommaSeparated({0})".format(self._item_validator)


class Regex(ScalarValidator):
    def __init__(self, regular_expression):
        """
        Give regular expression, e.g. u'[0-9]'
        """
        self._regex = regular_expression
        self._matching_message = "when expecting string matching {0}".format(
            self._regex
        )

    def validate_scalar(self, chunk):
        if re.compile(self._regex).match(chunk.contents) is None:
            chunk.expecting_but_found(
                self._matching_message, "found non-matching string"
            )
        return chunk.contents

    def to_yaml(self, data):
        self.should_be_string(data, self._matching_message)
        if re.compile(self._regex).match(data) is None:
            raise YAMLSerializationError(
                "{} found '{}'".format(self._matching_message, data)
            )
        return data


class Email(Regex):
    def __init__(self):
        self._regex = constants.REGEXES["email"]
        self._matching_message = "when expecting an email address"


class Url(Regex):
    def __init__(self):
        self._regex = constants.REGEXES["url"]
        self._matching_message = "when expecting a url"


class Str(ScalarValidator):
    def validate_scalar(self, chunk):
        return chunk.contents

    def to_yaml(self, data):
        if not utils.is_string(data):
            raise YAMLSerializationError("'{}' is not a string".format(data))
        if "\n" in data:
            return PreservedScalarString(data)
        return data


class Int(ScalarValidator):
    def validate_scalar(self, chunk):
        val = chunk.contents
        if not utils.is_integer(val):
{'fraction_non_alphanumeric': 0.09535548762449463, 'fraction_numerical': 0.003451336160141998, 'mean_word_length': 2.390839184219325, 'pattern_counts': {'":': 2, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6303905', 'n_tokens_mistral': 2855, 'n_tokens_neox': 2784, 'n_words': 689}