<commit_before>"""
Functions for validating JSON against the pScheduler JSON and standard
values dictionaries
"""

import copy
import jsonschema

# TODO: Consider adding tile/description and maybe "example" (not
# officially supported) as a way to generate the JSON dictionary.


# Note that adding an "x-invalid-message" string to any type will use
# that value for error messages instead of jsonschema's default.  The
# sequence "%s" in that string will be replaced with the invalid value.
#
# See the definition of "Duration" for an example.


#
# Types from the dictionary
#
__dictionary__ = {
    
    #
    # JSON Types
    #

    "AnyJSON": {
        "oneOf": [
            { "type": "array" },
            { "type": "boolean" },
            { "type": "integer" },
            { "type": "null" },
            { "type": "number" },
            { "type": "object" },
            { "type": "string" }
            ]
        },

    "Array": { "type": "array" },

    "AS": {
        "type": "object",
        "properties": {            
            "number": { "$ref": "#/pScheduler/Cardinal" },
            "owner": { "type": "string" },
            },
        "additionalProperties": False,
        "required": [ "number" ]
        },

    "Boolean": { "type": "boolean" },

    "Cardinal": {
        "type": "integer",
        "minimum": 1,
    },

    "CardinalList": {
        "type": "array",
        "items": { "$ref": "#/pScheduler/Cardinal" },
    },

    "CardinalRange": {
        "type": "object",
        "properties": {
            "lower": { "$ref": "#/pScheduler/Cardinal" },
            "upper": { "$ref": "#/pScheduler/Cardinal" }
        },
        "additionalProperties": False,
        "required": [ "lower", "upper" ]
    },

    "CardinalZero": {
        "type": "integer",
        "minimum": 0,
        },

    "CardinalZeroList": {
        "type": "array",
        "items": { "$ref": "#/pScheduler/CardinalZero" },
    },

    "CardinalZeroRange": {
        "type": "object",
        "properties": {
            "lower": { "$ref": "#/pScheduler/CardinalZero" },
            "upper": { "$ref": "#/pScheduler/CardinalZero" }
        },
        "additionalProperties": False,
        "required": [ "lower", "upper" ]
    },

    "ClockState": {
        "type": "object",
        "properties": {
            "time":         { "$ref": "#/pScheduler/Timestamp" },
            "synchronized": { "$ref": "#/pScheduler/Boolean" },
            "source":       { "$ref": "#/pScheduler/String" },
            "reference":    { "$ref": "#/pScheduler/String" },
            "offset":       { "$ref": "#/pScheduler/Number" },
        },
        "additionalProperties": False,
        "required": [ "time", "synchronized" ]
    },

    "Duration": {
        "type": "string",
        # ISO 8601.  Source: https://gist.github.com/philipashlock/8830168
        # Modified not to accept repeats (e.g., R5PT1M), which we don't support.
        # Modified not to accept months or years, which are inexact.
        "pattern": r'^P(?:\d+(?:\.\d+)?W)?(?:\d+(?:\.\d+)?D)?(?:T(?:\d+(?:\.\d+)?H)?(?:\d+(?:\.\d+)?M)?(?:\d+(?:\.\d+)?S)?)?$',
        "x-invalid-message": "'%s' is not a valid ISO 8601 duration."
        },

    "DurationRange": {
        "type": "object",
        "properties": {
            "lower": { "$ref": "#/pScheduler/Duration" },
            "upper": { "$ref": "#/pScheduler/Duration" }
        },
        "additionalProperties": False,
        "required": [ "lower", "upper" ]
    },

    "Email": { "type": "string", "format": "email" },

    "Float": {
        "type": "number"
    },

    "GeographicPosition": {
        "type": "string",
        # ISO 6709
        # Source:  https://svn.apache.org/repos/asf/abdera/abdera2/common/src/main/java/org/apache/abdera2/common/geo/IsoPosition.java
        "pattern": r'^(([+-]\d{2})(\d{2})?(\d{2})?(\.\d+)?)(([+-]\d{3})(\d{2})?(\d{2})?(\.\d+)?)([+-]\d+(\.\d+)?)?$'
        },

    "Host": {
        "anyOf": [
            { "$ref": "#/pScheduler/HostName" },
            { "$ref": "#/pScheduler/IPAddress" },
        ]
    },

    "HostName": {
        "type": "string",
        "format": "host-name"
        },

    "Integer": { "type": "integer" },

    "IPAddress": {
        "oneOf": [
            { "type": "string", "format": "ipv4" },
            { "type": "string", "format": "ipv6" },
            ]
        },

    "IPv4": { "type": "string", "format": "ipv4" },

    "IPv6": { "type": "string", "format": "ipv6" },

    "IPv4CIDR": {
        "type": "string",
        # Source: http://blog.markhatton.co.uk/2011/03/15/regular-expressions-for-ip-addresses-cidr-ranges-and-hostnames
        "pattern":r'^(([0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])\.){3}([0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])(\/([0-9]|[1-2][0-9]|3[0-2]))$'
        },

    "IPv6CIDR": {
        "type": "string",
        # Source: http://www.regexpal.com/93988
        "pattern": r'^s*((([0-9A-Fa-f]{1,4}:){7}([0-9A-Fa-f]{1,4}|:))|(([0-9A-Fa-f]{1,4}:){6}(:[0-9A-Fa-f]{1,4}|((25[0-5]|2[0-4]d|1dd|[1-9]?d)(.(25[0-5]|2[0-4]d|1dd|[1-9]?d)){3})|:))|(([0-9A-Fa-f]{1,4}:){5}(((:[0-9A-Fa-f]{1,4}){1,2})|:((25[0-5]|2[0-4]d|1dd|[1-9]?d)(.(25[0-5]|2[0-4]d|1dd|[1-9]?d)){3})|:))|(([0-9A-Fa-f]{1,4}:){4}(((:[0-9A-Fa-f]{1,4}){1,3})|((:[0-9A-Fa-f]{1,4})?:((25[0-5]|2[0-4]d|1dd|[1-9]?d)(.(25[0-5]|2[0-4]d|1dd|[1-9]?d)){3}))|:))|(([0-9A-Fa-f]{1,4}:){3}(((:[0-9A-Fa-f]{1,4}){1,4})|((:[0-9A-Fa-f]{1,4}){0,2}:((25[0-5]|2[0-4]d|1dd|[1-9]?d)(.(25[0-5]|2[0-4]d|1dd|[1-9]?d)){3}))|:))|(([0-9A-Fa-f]{1,4}:){2}(((:[0-9A-Fa-f]{1,4}){1,5})|((:[0-9A-Fa-f]{1,4}){0,3}:((25[0-5]|2[0-4]d|1dd|[1-9]?d)(.(25[0-5]|2[0-4]d|1dd|[1-9]?d)){3}))|:))|(([0-9A-Fa-f]{1,4}:){1}(((:[0-9A-Fa-f]{1,4}){1,6})|((:[0-9A-Fa-f]{1,4}){0,4}:((25[0-5]|2[0-4]d|1dd|[1-9]?d)(.(25[0-5]|2[0-4]d|1dd|[1-9]?d)){3}))|:))|(:(((:[0-9A-Fa-f]{1,4}){1,7})|((:[0-9A-Fa-f]{1,4}){0,5}:((25[0-5]|2[0-4]d|1dd|[1-9]?d)(.(25[0-5]|2[0-4]d|1dd|[1-9]?d)){3}))|:)))(%.+)?s*(\/([0-9]|[1-9][0-9]|1[0-1][0-9]|12[0-8]))?$'
    },

    "IPCIDR": {
        "oneOf": [
            { "$ref": "#/pScheduler/IPv4CIDR" },
            { "$ref": "#/pScheduler/IPv6CIDR" },
        ]
    },

    "Int8": {
        "type": "integer",
        "minimum": -128,
        "maximum": 127
    },

    "UInt8": {
        "type": "integer",
        "minimum": 0,
        "maximum": 255
    },

    "Int16": {
        "type": "integer",
        "minimum": -32768,
        "maximum": 32767
    },

    "UInt16": {
        "type": "integer",
        "minimum": 0,
        "maximum": 65535
    },

    "Int32": {
        "type": "integer",
        "minimum": -2147483648,
        "maximum": 2147483647
    },

    "UInt32": {
        "type": "integer",
        "minimum": 0,
        "maximum": 4294967295
    },

    "Int64": {
        "type": "integer",
        "minimum": -9223372036854775808,
        "maximum": 9223372036854775807
    },

    "UInt64": {
        "type": "integer",
        "minimum": 0,
        "maximum": 184446744073709551615
        },


    "IPPort": {
        "type": "integer",
        "minimum": 0,
        "maximum": 65535
        },
    
    "IPPortRange": {
        "type": "object",
        "properties": {
            "lower": { "$ref": "#/pScheduler/IPPort" },
            "upper": { "$ref": "#/pScheduler/IPPort" }
        },
        "additionalProperties": False,
        "required": [ "lower", "upper" ]
    },
    
    "IPTOS": {
        "type": "integer",
        "minimum": 0,
        "maximum": 255
        },

    "JQTransformSpecification": {
        "type": "object",
        "properties": {
            "script":    { "$ref": "#/pScheduler/String" },
            "output-raw": { "$ref": "#/pScheduler/Boolean" }
        },
        "additionalProperties": False,
        "required": [ "script" ]
    },
    
    "Number": { "type": "number" },

    "Numeric": {
        "anyOf": [
            { "$ref": "#/pScheduler/Number" },
            { "$ref": "#/pScheduler/SINumber" },
            ]
    },

    "NumericRange": {
        "type": "object",
        "properties": {
            "lower": { "$ref": "#/pScheduler/Numeric" },
            "upper": { "$ref": "#/pScheduler/Numeric" }
        },
        "additionalProperties": False,
        "required": [ "lower", "upper" ]<commit_msg>Added args to the JQTransformSpecification as an AnyJSON<commit_after>"""
Functions for validating JSON against the pScheduler JSON and standard
values dictionaries
"""

import copy
import jsonschema

# TODO: Consider adding tile/description and maybe "example" (not
# officially supported) as a way to generate the JSON dictionary.


# Note that adding an "x-invalid-message" string to any type will use
# that value for error messages instead of jsonschema's default.  The
# sequence "%s" in that string will be replaced with the invalid value.
#
# See the definition of "Duration" for an example.


#
# Types from the dictionary
#
__dictionary__ = {
    
    #
    # JSON Types
    #

    "AnyJSON": {
        "oneOf": [
            { "type": "array" },
            { "type": "boolean" },
            { "type": "integer" },
            { "type": "null" },
            { "type": "number" },
            { "type": "object" },
            { "type": "string" }
            ]
        },

    "Array": { "type": "array" },

    "AS": {
        "type": "object",
        "properties": {            
            "number": { "$ref": "#/pScheduler/Cardinal" },
            "owner": { "type": "string" },
            },
        "additionalProperties": False,
        "required": [ "number" ]
        },

    "Boolean": { "type": "boolean" },

    "Cardinal": {
        "type": "integer",
        "minimum": 1,
    },

    "CardinalList": {
        "type": "array",
        "items": { "$ref": "#/pScheduler/Cardinal" },
    },

    "CardinalRange": {
        "type": "object",
        "properties": {
            "lower": { "$ref": "#/pScheduler/Cardinal" },
            "upper": { "$ref": "#/pScheduler/Cardinal" }
        },
        "additionalProperties": False,
        "required": [ "lower", "upper" ]
    },

    "CardinalZero": {
        "type": "integer",
        "minimum": 0,
        },

    "CardinalZeroList": {
        "type": "array",
        "items": { "$ref": "#/pScheduler/CardinalZero" },
    },

    "CardinalZeroRange": {
        "type": "object",
        "properties": {
            "lower": { "$ref": "#/pScheduler/CardinalZero" },
            "upper": { "$ref": "#/pScheduler/CardinalZero" }
        },
        "additionalProperties": False,
        "required": [ "lower", "upper" ]
    },

    "ClockState": {
        "type": "object",
        "properties": {
            "time":         { "$ref": "#/pScheduler/Timestamp" },
            "synchronized": { "$ref": "#/pScheduler/Boolean" },
            "source":       { "$ref": "#/pScheduler/String" },
            "reference":    { "$ref": "#/pScheduler/String" },
            "offset":       { "$ref": "#/pScheduler/Number" },
        },
        "additionalProperties": False,
        "required": [ "time", "synchronized" ]
    },

    "Duration": {
        "type": "string",
        # ISO 8601.  Source: https://gist.github.com/philipashlock/8830168
        # Modified not to accept repeats (e.g., R5PT1M), which we don't support.
        # Modified not to accept months or years, which are inexact.
        "pattern": r'^P(?:\d+(?:\.\d+)?W)?(?:\d+(?:\.\d+)?D)?(?:T(?:\d+(?:\.\d+)?H)?(?:\d+(?:\.\d+)?M)?(?:\d+(?:\.\d+)?S)?)?$',
        "x-invalid-message": "'%s' is not a valid ISO 8601 duration."
        },

    "DurationRange": {
        "type": "object",
        "properties": {
            "lower": { "$ref": "#/pScheduler/Duration" },
            "upper": { "$ref": "#/pScheduler/Duration" }
        },
        "additionalProperties": False,
        "required": [ "lower", "upper" ]
    },

    "Email": { "type": "string", "format": "email" },

    "Float": {
        "type": "number"
    },

    "GeographicPosition": {
        "type": "string",
        # ISO 6709
        # Source:  https://svn.apache.org/repos/asf/abdera/abdera2/common/src/main/java/org/apache/abdera2/common/geo/IsoPosition.java
        "pattern": r'^(([+-]\d{2})(\d{2})?(\d{2})?(\.\d+)?)(([+-]\d{3})(\d{2})?(\d{2})?(\.\d+)?)([+-]\d+(\.\d+)?)?$'
        },

    "Host": {
        "anyOf": [
            { "$ref": "#/pScheduler/HostName" },
            { "$ref": "#/pScheduler/IPAddress" },
        ]
    },

    "HostName": {
        "type": "string",
        "format": "host-name"
        },

    "Integer": { "type": "integer" },

    "IPAddress": {
        "oneOf": [
            { "type": "string", "format": "ipv4" },
            { "type": "string", "format": "ipv6" },
            ]
        },

    "IPv4": { "type": "string", "format": "ipv4" },

    "IPv6": { "type": "string", "format": "ipv6" },

    "IPv4CIDR": {
        "type": "string",
        # Source: http://blog.markhatton.co.uk/2011/03/15/regular-expressions-for-ip-addresses-cidr-ranges-and-hostnames
        "pattern":r'^(([0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])\.){3}([0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])(\/([0-9]|[1-2][0-9]|3[0-2]))$'
        },

    "IPv6CIDR": {
        "type": "string",
        # Source: http://www.regexpal.com/93988
        "pattern": r'^s*((([0-9A-Fa-f]{1,4}:){7}([0-9A-Fa-f]{1,4}|:))|(([0-9A-Fa-f]{1,4}:){6}(:[0-9A-Fa-f]{1,4}|((25[0-5]|2[0-4]d|1dd|[1-9]?d)(.(25[0-5]|2[0-4]d|1dd|[1-9]?d)){3})|:))|(([0-9A-Fa-f]{1,4}:){5}(((:[0-9A-Fa-f]{1,4}){1,2})|:((25[0-5]|2[0-4]d|1dd|[1-9]?d)(.(25[0-5]|2[0-4]d|1dd|[1-9]?d)){3})|:))|(([0-9A-Fa-f]{1,4}:){4}(((:[0-9A-Fa-f]{1,4}){1,3})|((:[0-9A-Fa-f]{1,4})?:((25[0-5]|2[0-4]d|1dd|[1-9]?d)(.(25[0-5]|2[0-4]d|1dd|[1-9]?d)){3}))|:))|(([0-9A-Fa-f]{1,4}:){3}(((:[0-9A-Fa-f]{1,4}){1,4})|((:[0-9A-Fa-f]{1,4}){0,2}:((25[0-5]|2[0-4]d|1dd|[1-9]?d)(.(25[0-5]|2[0-4]d|1dd|[1-9]?d)){3}))|:))|(([0-9A-Fa-f]{1,4}:){2}(((:[0-9A-Fa-f]{1,4}){1,5})|((:[0-9A-Fa-f]{1,4}){0,3}:((25[0-5]|2[0-4]d|1dd|[1-9]?d)(.(25[0-5]|2[0-4]d|1dd|[1-9]?d)){3}))|:))|(([0-9A-Fa-f]{1,4}:){1}(((:[0-9A-Fa-f]{1,4}){1,6})|((:[0-9A-Fa-f]{1,4}){0,4}:((25[0-5]|2[0-4]d|1dd|[1-9]?d)(.(25[0-5]|2[0-4]d|1dd|[1-9]?d)){3}))|:))|(:(((:[0-9A-Fa-f]{1,4}){1,7})|((:[0-9A-Fa-f]{1,4}){0,5}:((25[0-5]|2[0-4]d|1dd|[1-9]?d)(.(25[0-5]|2[0-4]d|1dd|[1-9]?d)){3}))|:)))(%.+)?s*(\/([0-9]|[1-9][0-9]|1[0-1][0-9]|12[0-8]))?$'
    },

    "IPCIDR": {
        "oneOf": [
            { "$ref": "#/pScheduler/IPv4CIDR" },
            { "$ref": "#/pScheduler/IPv6CIDR" },
        ]
    },

    "Int8": {
        "type": "integer",
        "minimum": -128,
        "maximum": 127
    },

    "UInt8": {
        "type": "integer",
        "minimum": 0,
        "maximum": 255
    },

    "Int16": {
        "type": "integer",
        "minimum": -32768,
        "maximum": 32767
    },

    "UInt16": {
        "type": "integer",
        "minimum": 0,
        "maximum": 65535
    },

    "Int32": {
        "type": "integer",
        "minimum": -2147483648,
        "maximum": 2147483647
    },

    "UInt32": {
        "type": "integer",
        "minimum": 0,
        "maximum": 4294967295
    },

    "Int64": {
        "type": "integer",
        "minimum": -9223372036854775808,
        "maximum": 9223372036854775807
    },

    "UInt64": {
        "type": "integer",
        "minimum": 0,
        "maximum": 184446744073709551615
        },


    "IPPort": {
        "type": "integer",
        "minimum": 0,
        "maximum": 65535
        },
    
    "IPPortRange": {
        "type": "object",
        "properties": {
            "lower": { "$ref": "#/pScheduler/IPPort" },
            "upper": { "$ref": "#/pScheduler/IPPort" }
        },
        "additionalProperties": False,
        "required": [ "lower", "upper" ]
    },
    
    "IPTOS": {
        "type": "integer",
        "minimum": 0,
        "maximum": 255
        },

    "JQTransformSpecification": {
        "type": "object",
        "properties": {
            "script":    { "$ref": "#/pScheduler/String" },
            "output-raw": { "$ref": "#/pScheduler/Boolean" },
			"args": { "$ref": "#/pScheduler/AnyJSON" }
        },
        "additionalProperties": False,
        "required": [ "script" ]
    },
    
    "Number": { "type": "number" },

    "Numeric": {
        "anyOf": [
            { "$ref": "#/pScheduler/Number" },
            { "$ref": "#/pScheduler/SINumber" },
            ]
    },

    "NumericRange": {
        "type": "object",
        "properties": {
            "lower": { "$ref": "#/pScheduler/Numeric" },
            "upper": { "$ref": "#/pScheduler/Numeric" }
        },
        "additionalProperties": False,
        "required": [ "lower", "upper" ]
{'fraction_non_alphanumeric': 0.2692630820931349, 'fraction_numerical': 0.061449831973115696, 'mean_word_length': 2.0929844097995547, 'pattern_counts': {'":': 390, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 4, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6534830', 'n_tokens_mistral': 7084, 'n_tokens_neox': 6599, 'n_words': 894}