<commit_before>/**
 * @module datatypes/xmlschema
 * @desc Implementation of the XMLSchema datatypes.
 * @author <NAME>
 * @license MPL 2.0
 * @copyright 2013, 2014 Mangalam Research Center for Buddhist Languages
 */
define(/** @lends module:datatypes/builtin */
    function (require, exports, module) {
'use strict';

var _ = require("lodash");
var errors = require("./errors");
var regexp = require("./regexp");
var xmlcharacters = require("./xmlcharacters");

// Whitespace processing.
var PRESERVE = 1;
var REPLACE = 2;
var COLLAPSE = 3;
var xml_Name_re = xmlcharacters.xml_Name_re;
var xml_NCName = xmlcharacters.xml_NCName;
var xml_NCName_re = xmlcharacters.xml_NCName_re;
var xml_Name_Char = xmlcharacters.xml_Name_Char;
var xml_Letter = xmlcharacters.xml_Letter;
var ParamError = errors.ParamError;
var ValueError = errors.ValueError;
var ParameterParsingError = errors.ParameterParsingError;
var ValueValidationError = errors.ValueValidationError;

function failIfNotInteger(value, name) {
    if (value.search(/^\d+$/) !== -1)
        return false;

    return new ParamError(name + " must have an integer value");
}

function failIfNotNonNegativeInteger(value, name) {
    if (!failIfNotInteger(value, name) && Number(value) >= 0)
            return false;

    return new ParamError(name + " must have a non-negative integer value");
}

function failIfNotPositiveInteger(value, name) {
    if (!failIfNotInteger(value, name) && Number(value) > 0)
        return false;

    return new ParamError(name + " must have a positive value");
}

//
// The parameters
//

/**
 * @classdesc This class does not exist as a JavaScript entity. This
 * is a pseudo-class describing the structure that all parameter types
 * used for XML Schema type processing share.
 *
 * @name module:datatypes/xmlschema~Parameter
 * @class
 * @private
 *
 * @property {string} name Name of the parameter.
 * @property {boolean} repeatable Whether it can appear more than once
 * on the same type.
 */

/**
 * Convert the parameter value from a string to a value to be used
 * internally by this code.
 *
 * @method
 * @name module:datatypes/xmlschema~Parameter#convert
 * @param {string} value Value to convert.
 * @returns The converted value.
 */


/**
 * Checks whether a parameter is invalid.
 *
 * @method
 * @name module:datatypes/xmlschema~Parameter#isInvalidParam
 * @param {string} value The parameter value to check. This is the raw
 * string from the schema, not a value converted by {@link
 * module:datatypes/xmlschema~Parameter#convert convert}.
 * @param {string} name The name of the parameter. This allows using
 * generic functions to check on values.
 * @param {string} type The {@link module:datatypes~Datatype
 * Datatype} object for which this parameter is checked.
 * @returns {false|Array.<module:datatypes/errors~ParamError>}
 * ``false`` if there is no problem. Otherwise, an array of errors.
 */

/**
 * Checks whether a value that appears in the XML document being
 * validated is invalid according to this parameter.
 *
 * @method
 * @name module:datatypes/xmlschema~Parameter#isInvalidValue
 * @param {Object} value The value from the XML document. This is the
 * parsed value, converted by {@link
 * module:datatypes~Datatype#parseValue parseValue}.
 * @param param The parameter value. This must be the value obtained
 * from {@link module:datatypes/xmlschema~Parameter#convert convert}.
 * @param {string} type The {@link module:datatypes~Datatype
 * Datatype} object for which this parameter is checked.
 * @returns {false|module:datatypes/errors~ValueError} ``false`` if there is no
 * problem. Otherwise, an error.
 */

var length_p = {
    name: "length",
    repeatable: false,
    convert: Number,
    isInvalidParam: failIfNotNonNegativeInteger,
    isInvalidValue: function (value, param, type) {
        if (type.valueLength(value) === param)
            return false;

        return new ValueError("length of value should be " + param);
    }
};

var minLength_p = {
    name: "minLength",
    repeatable: false,
    convert: Number,
    isInvalidParam: failIfNotNonNegativeInteger,
    isInvalidValue: function (value, param, type) {
        if (type.valueLength(value) >= param)
            return false;

        return new ValueError("length of value should be greater than "+
                              "or equal to " + param);
    }
};

var maxLength_p = {
    name: "maxLength",
    repeatable: false,
    convert: Number,
    isInvalidParam: failIfNotNonNegativeInteger,
    isInvalidValue: function (value, param, type) {
        if (type.valueLength(value) <= param)
            return false;

        return new ValueError("length of value should be less than "+
                              "or equal to " + param);
    }
};

//
// pattern is special. It converts the param value found in the RNG
// file into an object with two fields: ``rng`` and ``internal``. RNG
// is the string value from the RNG file, and ``internal`` is a
// representation internal to salve. We use ``internal`` for
// performing the validation but present ``rng`` to the user. Note
// that if pattern appears multiple times as a parameter, the two
// values are the result of the concatenation of all the instance of
// the pattern parameter. (Why this? Because it would be confusing to
// show the internal value in error messages to the user.)
//

var multi_char_escapes = [
    /\\s/g, "[ \\t\\n\\r]",
    /\\S/g, "[^ \\t\\n\\r]",
    /\\i/g, "[" + xml_Letter + "_:]",
    /\\I/g, "[^" + xml_Letter + "_:]",
    /\\c/g, "[" + xml_Name_Char + "]",
    /\\C/g, "[^" + xml_Name_Char + "]",
    /\\d/g, "\\p{Nd}",
    /\\D/g, "[^\\d]",
    /\\w/g, "[^\\p{P}\\p{Z}\\p{C}]",
    /\\W/g, "[\\p{P}\\p{Z}\\p{C}]"
];

var pattern_p = {
    name: "pattern",
    repeatable: "combine",
    convert: function (value) {
        return {rng: value, internal: regexp.parse(value)};
    },
    combine: function (values) {
        return _.map(values, this.convert);
    },
    isInvalidParam: function (value) {
        try {
            this.convert(value);
        }
        catch (ex) {
            // Convert the error into something that makes sense for salve.
            if (ex instanceof regexp.SalveParsingError)
                return new ParamError(ex.message);

            // Rethrow
            throw ex;
        }
        return false;
    },
    isInvalidValue: function (value, param, type) {
        if (param instanceof Array) {
            var failed_on;
            for(var i = 0; !failed_on && i < param.length; ++i)
                if (!param[i].internal.test(value))
                    failed_on = param[i];

            if (!failed_on)
                return false;<commit_msg>Cache the results of regexp processing.<commit_after>/**
 * @module datatypes/xmlschema
 * @desc Implementation of the XMLSchema datatypes.
 * @author <NAME>
 * @license MPL 2.0
 * @copyright 2013, 2014 Mangalam Research Center for Buddhist Languages
 */
define(/** @lends module:datatypes/builtin */
    function (require, exports, module) {
'use strict';

var _ = require("lodash");
var errors = require("./errors");
var regexp = require("./regexp");
var xmlcharacters = require("./xmlcharacters");

// Whitespace processing.
var PRESERVE = 1;
var REPLACE = 2;
var COLLAPSE = 3;
var xml_Name_re = xmlcharacters.xml_Name_re;
var xml_NCName = xmlcharacters.xml_NCName;
var xml_NCName_re = xmlcharacters.xml_NCName_re;
var xml_Name_Char = xmlcharacters.xml_Name_Char;
var xml_Letter = xmlcharacters.xml_Letter;
var ParamError = errors.ParamError;
var ValueError = errors.ValueError;
var ParameterParsingError = errors.ParameterParsingError;
var ValueValidationError = errors.ValueValidationError;

function failIfNotInteger(value, name) {
    if (value.search(/^\d+$/) !== -1)
        return false;

    return new ParamError(name + " must have an integer value");
}

function failIfNotNonNegativeInteger(value, name) {
    if (!failIfNotInteger(value, name) && Number(value) >= 0)
            return false;

    return new ParamError(name + " must have a non-negative integer value");
}

function failIfNotPositiveInteger(value, name) {
    if (!failIfNotInteger(value, name) && Number(value) > 0)
        return false;

    return new ParamError(name + " must have a positive value");
}

//
// The parameters
//

/**
 * @classdesc This class does not exist as a JavaScript entity. This
 * is a pseudo-class describing the structure that all parameter types
 * used for XML Schema type processing share.
 *
 * @name module:datatypes/xmlschema~Parameter
 * @class
 * @private
 *
 * @property {string} name Name of the parameter.
 * @property {boolean} repeatable Whether it can appear more than once
 * on the same type.
 */

/**
 * Convert the parameter value from a string to a value to be used
 * internally by this code.
 *
 * @method
 * @name module:datatypes/xmlschema~Parameter#convert
 * @param {string} value Value to convert.
 * @returns The converted value.
 */


/**
 * Checks whether a parameter is invalid.
 *
 * @method
 * @name module:datatypes/xmlschema~Parameter#isInvalidParam
 * @param {string} value The parameter value to check. This is the raw
 * string from the schema, not a value converted by {@link
 * module:datatypes/xmlschema~Parameter#convert convert}.
 * @param {string} name The name of the parameter. This allows using
 * generic functions to check on values.
 * @param {string} type The {@link module:datatypes~Datatype
 * Datatype} object for which this parameter is checked.
 * @returns {false|Array.<module:datatypes/errors~ParamError>}
 * ``false`` if there is no problem. Otherwise, an array of errors.
 */

/**
 * Checks whether a value that appears in the XML document being
 * validated is invalid according to this parameter.
 *
 * @method
 * @name module:datatypes/xmlschema~Parameter#isInvalidValue
 * @param {Object} value The value from the XML document. This is the
 * parsed value, converted by {@link
 * module:datatypes~Datatype#parseValue parseValue}.
 * @param param The parameter value. This must be the value obtained
 * from {@link module:datatypes/xmlschema~Parameter#convert convert}.
 * @param {string} type The {@link module:datatypes~Datatype
 * Datatype} object for which this parameter is checked.
 * @returns {false|module:datatypes/errors~ValueError} ``false`` if there is no
 * problem. Otherwise, an error.
 */

var length_p = {
    name: "length",
    repeatable: false,
    convert: Number,
    isInvalidParam: failIfNotNonNegativeInteger,
    isInvalidValue: function (value, param, type) {
        if (type.valueLength(value) === param)
            return false;

        return new ValueError("length of value should be " + param);
    }
};

var minLength_p = {
    name: "minLength",
    repeatable: false,
    convert: Number,
    isInvalidParam: failIfNotNonNegativeInteger,
    isInvalidValue: function (value, param, type) {
        if (type.valueLength(value) >= param)
            return false;

        return new ValueError("length of value should be greater than "+
                              "or equal to " + param);
    }
};

var maxLength_p = {
    name: "maxLength",
    repeatable: false,
    convert: Number,
    isInvalidParam: failIfNotNonNegativeInteger,
    isInvalidValue: function (value, param, type) {
        if (type.valueLength(value) <= param)
            return false;

        return new ValueError("length of value should be less than "+
                              "or equal to " + param);
    }
};

//
// pattern is special. It converts the param value found in the RNG
// file into an object with two fields: ``rng`` and ``internal``. RNG
// is the string value from the RNG file, and ``internal`` is a
// representation internal to salve. We use ``internal`` for
// performing the validation but present ``rng`` to the user. Note
// that if pattern appears multiple times as a parameter, the two
// values are the result of the concatenation of all the instance of
// the pattern parameter. (Why this? Because it would be confusing to
// show the internal value in error messages to the user.)
//

var multi_char_escapes = [
    /\\s/g, "[ \\t\\n\\r]",
    /\\S/g, "[^ \\t\\n\\r]",
    /\\i/g, "[" + xml_Letter + "_:]",
    /\\I/g, "[^" + xml_Letter + "_:]",
    /\\c/g, "[" + xml_Name_Char + "]",
    /\\C/g, "[^" + xml_Name_Char + "]",
    /\\d/g, "\\p{Nd}",
    /\\D/g, "[^\\d]",
    /\\w/g, "[^\\p{P}\\p{Z}\\p{C}]",
    /\\W/g, "[\\p{P}\\p{Z}\\p{C}]"
];

var re_cache = Object.create(null);

var pattern_p = {
    name: "pattern",
    repeatable: "combine",
    convert: function (value) {
        var internal = re_cache[value];
        if (internal === undefined)
            internal = re_cache[value] = regexp.parse(value);
        return {
            rng: value,
            internal: internal
        };
    },
    combine: function (values) {
        return _.map(values, this.convert);
    },
    isInvalidParam: function (value) {
        try {
            this.convert(value);
        }
        catch (ex) {
            // Convert the error into something that makes sense for salve.
            if (ex instanceof regexp.SalveParsingError)
                return new ParamError(ex.message);

            // Rethrow
            throw ex;
        }
        return false;
    },
    isInvalidValue: function (value, param, type) {
        if (param instanceof Array) {
            var failed_on;
            for(var i = 0; !failed_on && i < param.length; ++i)
                if (!param[i].internal.test(value))
                    failed_on = param[i];

            if (!failed_on)
                return false;
{'fraction_non_alphanumeric': 0.1223622508792497, 'fraction_numerical': 0.0024912075029308325, 'mean_word_length': 2.940242494226328, 'pattern_counts': {'":': 0, '<': 11, '<?xml version=': 0, '>': 13, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 56}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2114434', 'n_tokens_mistral': 4073, 'n_tokens_neox': 4021, 'n_words': 1344}