<commit_before>var dotty = require('dotty');
var async = require('async');
var crypto = require('crypto');

var Schemator = function(object){
    var self = this;

    // All sanitizators goes here
    this.sanitizators = {
        pbkdf2 : function(value, config, callback){
            if(typeof value === 'string')
                crypto.pbkdf2(value, config[0], config[1], config[2], function(err, buffer){
                    callback(err, buffer.toString('hex'));
                });
            else
                callback();
        },
        remove : function(value, config, callback){
            this.remove();
            callback();
        }
    };

    // All validators goes here
    this.validators = {
        required : function(value, config, callback){
            callback(undefined, value !== undefined ? undefined : config);
        },
        minLength : function(value, config, callback){
            if(!value || !value.toString || value.toString().length < config[0])
                return callback(undefined, config[1]);
            callback();
        },
        maxLength : function(value, config, callback){
            if(!value || !value.toString || value.toString().length > config[0])
                return callback(undefined, config[1]);
            callback();
        },
        email : function(value, config, callback){
            if(!/[a-z0-9!#$%&'*+\/=?^_`{|}~-]+(?:\.[a-z0-9!#$%&'*+\/=?^_`{|}~-]+)*@(?:[a-z0-9](?:[a-z0-9-]*[a-z0-9])?\.)+[a-z0-9](?:[a-z0-9-]*[a-z0-9])?/.test(value))
                return callback(undefined, config);

            callback();
        },
        regex : function(value, config, callback){
            if(!config[0].test(value))
                return callback(undefined, config[1]);
            callback();
        },
        equalTo : function(value, config, callback){
            if(value !== this.readObject(config[0]))
                return callback(undefined, config[1]);
            callback();
        },
        alphanumeric : function(value, config, callback){
            if(!/^[a-z0-9]+$/i.test(value))
                return callback(undefined, config);
            callback();
        },
        alpha : function(value, config, callback){
            if(!/^[a-zA-Z]+$/.test(value))
                return callback(undefined, config);
            callback();
        },
        numeric : function(value, config, callback){
            if(!/^[0-9]+$/.test(value))
                return callback(undefined, config);
            callback();
        },
        inList : function(value, config, callback){
            if(config[0].indexOf(value) === -1)
                return callback(undefined, config[1]);
            callback();
        },
        notInList : function(value, config, callback){
            if(config[0].indexOf(value) !== -1)
                return callback(undefined, config[1]);
            callback();
        },
        optional : function(value, config, callback){
            if(!value)
                this.clearErrors();
            callback();
        }
    };


    // Here we will store all rules to use later
    this.rules = {
        sanitize : [],
        validate : []
    };

    // Store the configuration
    this.object = object;

    // Generate the rules
    dotty.deepKeys(object).forEach(function(el){
        var method = el[el.length - 1];
        if(method === 'sanitize' || method === 'validate')
            Object.keys(dotty.get(object, el)).forEach(function(key){
                var newOne = el.slice(0, el.length - 1);
                newOne.push(key);
                self.rules[method].push(newOne);
            });
    });
};

Schemator.prototype.run = function(object, callback){
    var self = this;
    var invalid = [];
    var result = {};
    var current = null;
    var context = {
        invalid : function(message, path){
            if(current)
                invalid.push([current.join('.'), message]);
        },
        clearErrors : function(path){
            var from = path ? path : current;
            invalid.forEach(function(el, index){
                if(el[0] === from.join('.'))
                    invalid.splice(index, 1);
            });
        },
        write : function(value, path){
            var to = path ? path : current;
            dotty.put(result, to, value);
        },
        remove : function(path){
            var to = path ? path : current;
            dotty.remove(result, to);
        },
        read : function(path){
            var from = path ? path : current;
            return dotty.get(result, from);
        },
        readObject : function(path){
            var from = path ? path : current;
            return dotty.get(object, from);
        }
    };

    async.series([
        function(callback){
            async.eachSeries(self.rules.validate, function(inspect, callback){
                var rule = inspect.slice(0);
                var func = rule[rule.length - 1];
                current = rule.slice(0, rule.length - 1);
                if(!self.validators[func])
                    return callback(new Error('Unknown validator ' + func));

                rule.splice(rule.length - 1, 0, 'validate');
                self.validators[func].apply(context, [dotty.get(object, current), dotty.get(self.object, rule), function(err, invalid){
                    if(invalid)
                        context.invalid(invalid);
                    else
                        context.write(dotty.get(object, current));

                    callback(err);
                }]);
            }, function(err){
                callback(err);
            });
        },
        function(callback){
            if(invalid.length)
                return callback();

            async.eachSeries(self.rules.sanitize, function(inspect, callback){
                var rule = inspect.slice(0);
                var func = rule[rule.length - 1];
                current = rule.slice(0, rule.length - 1);
                if(!self.sanitizators[func])
                    return callback(new Error('Unknown sanitizator ' + func));

                rule.splice(rule.length - 1, 0, 'sanitize');
                self.sanitizators[func].apply(context, [dotty.get(object, current), dotty.get(self.object, rule), function(err, result){
                    if(result)
                        context.write(result);

                    callback(err);
                }]);
            }, function(err){
                callback(err);
            });
        }
    ], function(err){<commit_msg>optional won't set undefined value<commit_after>var dotty = require('dotty');
var async = require('async');
var crypto = require('crypto');

var Schemator = function(object){
    var self = this;

    // All sanitizators goes here
    this.sanitizators = {
        pbkdf2 : function(value, config, callback){
            if(typeof value === 'string')
                crypto.pbkdf2(value, config[0], config[1], config[2], function(err, buffer){
                    callback(err, buffer.toString('hex'));
                });
            else
                callback();
        },
        remove : function(value, config, callback){
            this.remove();
            callback();
        }
    };

    // All validators goes here
    this.validators = {
        required : function(value, config, callback){
            callback(undefined, value !== undefined ? undefined : config);
        },
        minLength : function(value, config, callback){
            if(!value || !value.toString || value.toString().length < config[0])
                return callback(undefined, config[1]);
            callback();
        },
        maxLength : function(value, config, callback){
            if(!value || !value.toString || value.toString().length > config[0])
                return callback(undefined, config[1]);
            callback();
        },
        email : function(value, config, callback){
            if(!/[a-z0-9!#$%&'*+\/=?^_`{|}~-]+(?:\.[a-z0-9!#$%&'*+\/=?^_`{|}~-]+)*@(?:[a-z0-9](?:[a-z0-9-]*[a-z0-9])?\.)+[a-z0-9](?:[a-z0-9-]*[a-z0-9])?/.test(value))
                return callback(undefined, config);

            callback();
        },
        regex : function(value, config, callback){
            if(!config[0].test(value))
                return callback(undefined, config[1]);
            callback();
        },
        equalTo : function(value, config, callback){
            if(value !== this.readObject(config[0]))
                return callback(undefined, config[1]);
            callback();
        },
        alphanumeric : function(value, config, callback){
            if(!/^[a-z0-9]+$/i.test(value))
                return callback(undefined, config);
            callback();
        },
        alpha : function(value, config, callback){
            if(!/^[a-zA-Z]+$/.test(value))
                return callback(undefined, config);
            callback();
        },
        numeric : function(value, config, callback){
            if(!/^[0-9]+$/.test(value))
                return callback(undefined, config);
            callback();
        },
        inList : function(value, config, callback){
            if(config[0].indexOf(value) === -1)
                return callback(undefined, config[1]);
            callback();
        },
        notInList : function(value, config, callback){
            if(config[0].indexOf(value) !== -1)
                return callback(undefined, config[1]);
            callback();
        },
        optional : function(value, config, callback){
            if(!value){
                this.clearErrors();
                this.remove();
            }
            callback(undefined, undefined, true);
        }
    };


    // Here we will store all rules to use later
    this.rules = {
        sanitize : [],
        validate : []
    };

    // Store the configuration
    this.object = object;

    // Generate the rules
    dotty.deepKeys(object).forEach(function(el){
        var method = el[el.length - 1];
        if(method === 'sanitize' || method === 'validate')
            Object.keys(dotty.get(object, el)).forEach(function(key){
                var newOne = el.slice(0, el.length - 1);
                newOne.push(key);
                self.rules[method].push(newOne);
            });
    });
};

Schemator.prototype.run = function(object, callback){
    var self = this;
    var invalid = [];
    var result = {};
    var current = null;
    var context = {
        invalid : function(message, path){
            if(current)
                invalid.push([current.join('.'), message]);
        },
        clearErrors : function(path){
            var from = path ? path : current;
            invalid.forEach(function(el, index){
                if(el[0] === from.join('.'))
                    invalid.splice(index, 1);
            });
        },
        write : function(value, path){
            var to = path ? path : current;
            dotty.put(result, to, value);
        },
        remove : function(path){
            var to = path ? path : current;
            dotty.remove(result, to);
        },
        read : function(path){
            var from = path ? path : current;
            return dotty.get(result, from);
        },
        readObject : function(path){
            var from = path ? path : current;
            return dotty.get(object, from);
        }
    };

    async.series([
        function(callback){
            async.eachSeries(self.rules.validate, function(inspect, callback){
                var rule = inspect.slice(0);
                var func = rule[rule.length - 1];
                current = rule.slice(0, rule.length - 1);
                if(!self.validators[func])
                    return callback(new Error('Unknown validator ' + func));

                rule.splice(rule.length - 1, 0, 'validate');
                self.validators[func].apply(context, [dotty.get(object, current), dotty.get(self.object, rule), function(err, invalid, skip){
                    if(invalid)
                        context.invalid(invalid);
                    else
                        if(!skip)
                            context.write(dotty.get(object, current));

                    callback(err);
                }]);
            }, function(err){
                callback(err);
            });
        },
        function(callback){
            if(invalid.length)
                return callback();

            async.eachSeries(self.rules.sanitize, function(inspect, callback){
                var rule = inspect.slice(0);
                var func = rule[rule.length - 1];
                current = rule.slice(0, rule.length - 1);
                if(!self.sanitizators[func])
                    return callback(new Error('Unknown sanitizator ' + func));

                rule.splice(rule.length - 1, 0, 'sanitize');
                self.sanitizators[func].apply(context, [dotty.get(object, current), dotty.get(self.object, rule), function(err, result){
                    if(result)
                        context.write(result);

                    callback(err);
                }]);
            }, function(err){
                callback(err);
            });
        }
    ], function(err){
{'fraction_non_alphanumeric': 0.14820100653496582, 'fraction_numerical': 0.008412829565086757, 'mean_word_length': 1.5922897196261683, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2583372', 'n_tokens_mistral': 3646, 'n_tokens_neox': 3489, 'n_words': 722}