<commit_before>from functools import reduce
from pathlib import Path
import re

from typing import Tuple, Dict

from strictyaml import (
    ScalarValidator, Optional, Any, Bool, Int, Float, Str, Map,
    MapPattern, Seq, FixedSeq, Validator, OrValidator, YAMLValidationError,
)

from strictyaml.parser import generic_load


class Literal(ScalarValidator):
    """Validator that only matches a literal string."""

    _expected: str

    def __init__(self, expected: str):
        super().__init__()
        self._expected = expected

    def validate_scalar(self, chunk):
        if self._expected != chunk.contents:
            chunk.expecting_but_found(f"when expecting {self._expected}", "found non-matching string")
        return chunk.contents


def Choice(*args):
    """Validator that matches a choice of several literal strings."""
    return reduce(OrValidator, map(Literal, args))


def Scalar():
    """Validator that matches integers and floats."""
    return Int() | Float()


def FileMapping(glob_allowed: bool):
    """Validator that matches a file mapping: a string or a mapping with
    source and target.
    """
    if glob_allowed:
        return Str() | Map({
            'source': Str(),
            Optional('target'): Str(),
            Optional('mode'): Choice('simple', 'glob')
        })

    return Str() | Map({
        'source': Str(),
        Optional('target'): Str(),
    })


def Regex():
    """Validator that matches a regex: a mapping with pattern and optional
    mode.
    """
    return Map({
        'pattern': Str(),
        Optional('mode'): Choice('first', 'last', 'all'),
    })


def NumberCapture():
    """Validator that matches a predefined integer or float capture."""
    return Map({
        'type': Choice('integer', 'float'),
        'name': Str(),
        'prefix': Str(),
        Optional('mode'): Choice('first', 'last', 'all'),
    })


def Capture():
    """Validator that matches any of the valid capture specs."""
    return First(
        "capture",
        Str(),
        Regex(),
        NumberCapture(),
    )


class First(Validator):
    """Validator that validates against a sequence of sub-validators,
    picking the first that matches.
    """

    _validators: Tuple[Validator]

    def __init__(self, name: str, *validators: Validator):
        self._name = name
        self._validators = validators

    def __call__(self, chunk):
        for validator in self._validators:
            try:
                result = validator(chunk)
                result._selected_validator = validator
                result._validator = self
                return result
            except YAMLValidationError as e:
                pass
        else:
            raise YAMLValidationError(
                f"failed to find a valid schema for {self._name}",
                "found invalid input", chunk
            )


class Type(ScalarValidator):
    """Validator that parses a type description."""

    scalar_regex = re.compile('^(int|integer|str|string|float|floating|double)$')

    scalar_map = {
        'int': int, 'integer': int,
        'str': str, 'string': str,
        'float': float, 'floating': float, 'double': float,
    }

    def validate_scalar(self, chunk):
        if self.scalar_regex.match(chunk.contents):
            return self.scalar_map[chunk.contents]
        chunk.expecting_but_found("when expecting a valid type description", "found invalid input")


CASE_SCHEMA = Map({
    Optional('parameters'): MapPattern(
        Str(),
        First(
            "parameter",
            Seq(Scalar()),
            Seq(Str()),
            Map({
                'type': Literal('uniform'),
                'interval': FixedSeq([Scalar(), Scalar()]),
                'num': Int(),
            }),
            Map({
                'type': Literal('graded'),
                'interval': FixedSeq([Scalar(), Scalar()]),
                'num': Int(),
                'grading': Scalar(),
            })
        ),
    ),
    Optional('evaluate'): MapPattern(Str(), Str()),
    Optional('constants'): MapPattern(Str(), Float() | Int() | Str()),
    Optional('templates'): Seq(FileMapping(glob_allowed=False)),
    Optional('prefiles'): Seq(FileMapping(glob_allowed=True)),
    Optional('postfiles'): Seq(FileMapping(glob_allowed=True)),
    Optional('script'): Seq(First(
        "script command",
        Str(),
        Seq(Str()),
        Map({
            'command': Str() | Seq(Str()),
            Optional('name'): Str(),
            Optional('capture'): Capture() | Seq(Capture()),<commit_msg>Constants: parse ints before floats<commit_after>from functools import reduce
from pathlib import Path
import re

from typing import Tuple, Dict

from strictyaml import (
    ScalarValidator, Optional, Any, Bool, Int, Float, Str, Map,
    MapPattern, Seq, FixedSeq, Validator, OrValidator, YAMLValidationError,
)

from strictyaml.parser import generic_load


class Literal(ScalarValidator):
    """Validator that only matches a literal string."""

    _expected: str

    def __init__(self, expected: str):
        super().__init__()
        self._expected = expected

    def validate_scalar(self, chunk):
        if self._expected != chunk.contents:
            chunk.expecting_but_found(f"when expecting {self._expected}", "found non-matching string")
        return chunk.contents


def Choice(*args):
    """Validator that matches a choice of several literal strings."""
    return reduce(OrValidator, map(Literal, args))


def Scalar():
    """Validator that matches integers and floats."""
    return Int() | Float()


def FileMapping(glob_allowed: bool):
    """Validator that matches a file mapping: a string or a mapping with
    source and target.
    """
    if glob_allowed:
        return Str() | Map({
            'source': Str(),
            Optional('target'): Str(),
            Optional('mode'): Choice('simple', 'glob')
        })

    return Str() | Map({
        'source': Str(),
        Optional('target'): Str(),
    })


def Regex():
    """Validator that matches a regex: a mapping with pattern and optional
    mode.
    """
    return Map({
        'pattern': Str(),
        Optional('mode'): Choice('first', 'last', 'all'),
    })


def NumberCapture():
    """Validator that matches a predefined integer or float capture."""
    return Map({
        'type': Choice('integer', 'float'),
        'name': Str(),
        'prefix': Str(),
        Optional('mode'): Choice('first', 'last', 'all'),
    })


def Capture():
    """Validator that matches any of the valid capture specs."""
    return First(
        "capture",
        Str(),
        Regex(),
        NumberCapture(),
    )


class First(Validator):
    """Validator that validates against a sequence of sub-validators,
    picking the first that matches.
    """

    _validators: Tuple[Validator]

    def __init__(self, name: str, *validators: Validator):
        self._name = name
        self._validators = validators

    def __call__(self, chunk):
        for validator in self._validators:
            try:
                result = validator(chunk)
                result._selected_validator = validator
                result._validator = self
                return result
            except YAMLValidationError as e:
                pass
        else:
            raise YAMLValidationError(
                f"failed to find a valid schema for {self._name}",
                "found invalid input", chunk
            )


class Type(ScalarValidator):
    """Validator that parses a type description."""

    scalar_regex = re.compile('^(int|integer|str|string|float|floating|double)$')

    scalar_map = {
        'int': int, 'integer': int,
        'str': str, 'string': str,
        'float': float, 'floating': float, 'double': float,
    }

    def validate_scalar(self, chunk):
        if self.scalar_regex.match(chunk.contents):
            return self.scalar_map[chunk.contents]
        chunk.expecting_but_found("when expecting a valid type description", "found invalid input")


CASE_SCHEMA = Map({
    Optional('parameters'): MapPattern(
        Str(),
        First(
            "parameter",
            Seq(Scalar()),
            Seq(Str()),
            Map({
                'type': Literal('uniform'),
                'interval': FixedSeq([Scalar(), Scalar()]),
                'num': Int(),
            }),
            Map({
                'type': Literal('graded'),
                'interval': FixedSeq([Scalar(), Scalar()]),
                'num': Int(),
                'grading': Scalar(),
            })
        ),
    ),
    Optional('evaluate'): MapPattern(Str(), Str()),
    Optional('constants'): MapPattern(Str(), Int() | Float() | Str()),
    Optional('templates'): Seq(FileMapping(glob_allowed=False)),
    Optional('prefiles'): Seq(FileMapping(glob_allowed=True)),
    Optional('postfiles'): Seq(FileMapping(glob_allowed=True)),
    Optional('script'): Seq(First(
        "script command",
        Str(),
        Seq(Str()),
        Map({
            'command': Str() | Seq(Str()),
            Optional('name'): Str(),
            Optional('capture'): Capture() | Seq(Capture()),
{'fraction_non_alphanumeric': 0.1368982792419952, 'fraction_numerical': 0.0, 'mean_word_length': 2.369908256880734, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6368945', 'n_tokens_mistral': 2476, 'n_tokens_neox': 2376, 'n_words': 711}