<commit_before>var tu = require('./tutils'), concat = Function.apply.bind(Array.prototype.concat, []),
    loaders = [],
    types = {load, list, add},
    api = {
        /**
         * @param template String - looks for a template named something.
         */

        addTemplate: add('Template'),
        loadTemplate: load('Template'),
        listTemplates: list('Template'),

        addType: add('Type'),
        loadType: load('Type'),
        listTypes: list('Type'),

        addSchema: add('Schema'),
        loadSchema: load('Schema'),
        listSchemas: list('Schema'),

        addValidator: add('Validator'),
        loadValidator: load('Validator'),
        listValidators: list('Validator'),

        addProcessor: add('Processor'),
        loadProcessor: load('Processor'),
        listProcessors: list('Processor'),

        addLoader(loader){
            if (tu.isArray(loader)) {
                return loader.map(function (v) {
                    return this.addLoader(v);
                }, this)
            }
            Object.keys(loader).forEach(function (key) {
                if (!(key in this)) {
                    var parts = /^(load|list)(.*)/.exec(key);
                    if (parts && parts.length > 2 && parts[1] in types) {
                        this[key] = types[parts[1]](parts[2]);
                    } else {
                        console.log('do not understand ' + key);
                    }
                }
            }, this);

            loaders.unshift(loader);
            console.log('added loader', loader, loaders);<commit_msg>removed console.log<commit_after>var tu = require('./tutils'), concat = Function.apply.bind(Array.prototype.concat, []),
    loaders = [],
    types = {load, list, add},
    api = {
        /**
         * @param template String - looks for a template named something.
         */

        addTemplate: add('Template'),
        loadTemplate: load('Template'),
        listTemplates: list('Template'),

        addType: add('Type'),
        loadType: load('Type'),
        listTypes: list('Type'),

        addSchema: add('Schema'),
        loadSchema: load('Schema'),
        listSchemas: list('Schema'),

        addValidator: add('Validator'),
        loadValidator: load('Validator'),
        listValidators: list('Validator'),

        addProcessor: add('Processor'),
        loadProcessor: load('Processor'),
        listProcessors: list('Processor'),

        addLoader(loader){
            if (tu.isArray(loader)) {
                return loader.map(function (v) {
                    return this.addLoader(v);
                }, this)
            }
            Object.keys(loader).forEach(function (key) {
                if (!(key in this)) {
                    var parts = /^(load|list)(.*)/.exec(key);
                    if (parts && parts.length > 2 && parts[1] in types) {
                        this[key] = types[parts[1]](parts[2]);
                    } else {
                        console.log('do not understand ' + key);
                    }
                }
            }, this);

            loaders.unshift(loader);
{'fraction_non_alphanumeric': 0.14690885914595284, 'fraction_numerical': 0.0025493945188017845, 'mean_word_length': 1.655668358714044, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6459201', 'n_tokens_mistral': 822, 'n_tokens_neox': 784, 'n_words': 172}