<commit_before>var verymodel = require('verymodel');
var uuid = require('node-uuid');
var async = require('async');

function makeModelLevely(mf) {
    mf.addDefinition({
        key: {
            private: true,
            derive: function () {
                if (this.__verymeta.parent) {
                    return this.__verymeta.parent.key + (this.__verymeta.childsep || '~') + this.__verymeta.prefix + this.keyname;
                } else {
                    return this.__verymeta.prefix + (this.__verymeta.sep || '!') + this.keyname;
                }
            },
            required: true
        },
        keyname: {
            private: true,
            default: function () { return uuid.v4(); },
            required: true
        },
    });

    mf.load = function (key, callback) {
        //if this isn't a child and the user didn't include the prefix, add it
        if (key.indexOf(mf.options.childsep) === -1 && key.indexOf(mf.options.prefix) !== 0)  {
            key = mf.options.prefix + (mf.options.sep || '!') + key;
        }
        mf.options.db.get(key, function (err, result) {
            var obj;
            if (!err) {
                obj = mf.create(result);
                obj.key = key;
                callback(err, obj);
            } else {
                callback(err);
            }
        });
    };
    
    mf.get = mf.load;

    mf.delete = function (key, callback) {
        mf.options.db.del(key, callback);
    };

    mf.update = function (key, updated_fields, callback) {
        mf.load(key, function (err, result) {
            if (!err && result) {
                var keys = Object.keys(updated_fields);
                for (var idx = 0; idx < keys.length; idx++) {
                    result[keys[idx]] = updated_fields[keys[idx]];
                }
                result.save(function (err) {
                    callback(err, result);
                });
            } else {
                callback(err);
            }
        });
    };

    mf.all = function (callback) {
        var objects = [];
        var err;
        var stream = mf.options.db.createReadStream({
            start : mf.options.prefix,
            end : mf.options.prefix + '~'
        });
        stream.on('data', function (entry) {
            if (entry.key.indexOf(mf.options.childsep || '~') == -1) {
                var inst = mf.create(entry.value);
                inst.key = entry.key;
                objects.push(inst);
            }
        });
        stream.on('error', function (error) {
            err = error;
            callback(err, null);
        });
        stream.on('close', function () {
            callback(err, objects);
        });
    };

    function indexName(field) {
        return 'index!' + field;
    }


    mf.getByIndex = function (field, value, callback, _keyfilter, limit) {
        if (this.definition[field].hasOwnProperty('index') && this.definition[field].index === true) {
            mf.options.db.get(indexName(field), function (err, index) {
                var keys;
                if (err || !index) index = {};
                if (index.hasOwnProperty(value)) {
                    keys = index[value];
                    if (_keyfilter) {
                        keys = keys.filter(function (key) {
                            return key.indexOf(_keyfilter) === 0;
                        });
                    }
                    if (limit) {
                        keys = keys.slice(0, limit);
                    }
                    async.map(keys, function (key, acb) {
                        mf.options.db.get(key, function (err, result) {
                            var obj;
                            if (!err) {
                                obj = mf.create(result);
                                obj.key = key;
                                acb(err, obj);
                            } else {
                                acb(err);
                            }
                        }.bind(this));
                    }.bind(this),
                    function (err, results) {
                        if (limit === 1) {
                            callback(err, results[0]);
                        } else {
                            callback(err, results);
                        }
                    });
                } else {
                    callback("no index for value");
                }
            }.bind(this));
        } else {
            callback("field does not have index");
        }
    };

    mf.findByIndex = function (field, value, callback, _keyfilter) {
        mf.getByIndex.call(this, field, value, callback, _keyfilter, 1);
    };

    mf.extendModel({
        save: function (callback) {
            async.waterfall([
                function (acb) {
                    this.__verymeta.db.put(this.key, this.toJSON(), acb);
                }.bind(this),
                function (acb) {
                    async.each(Object.keys(this.__verymeta.defs), function (field, scb) {
                        var ikey;
                        if (this.__verymeta.defs[field].hasOwnProperty('index') && this.__verymeta.defs[field].index === true) {
                            ikey = indexName(field);
                            this.__verymeta.db.get(ikey, function (err, obj) {
                                var objkeys, idx, kidx;
                                if (err || !obj) {
                                    obj = {};
                                }
                                objkeys = Object.keys(obj);
                                for (kidx in objkeys) {
                                    idx = obj[objkeys[kidx]].indexOf(this.key);
                                    if (idx !== -1) {
                                        obj[objkeys[kidx]].pop(idx);
                                    }
                                }
                                if (!obj.hasOwnProperty(field)) obj[this[field]] = [];<commit_msg>better indexes<commit_after>var verymodel = require('verymodel');
var uuid = require('node-uuid');
var async = require('async');
var crypto = require('crypto');

function makeModelLevely(mf) {
    mf.addDefinition({
        key: {
            private: true,
            derive: function () {
                if (this.__verymeta.parent) {
                    return this.__verymeta.parent.key + (this.__verymeta.childsep || '~') + this.__verymeta.prefix + this.keyname;
                } else {
                    return this.__verymeta.prefix + (this.__verymeta.sep || '!') + this.keyname;
                }
            },
            required: true
        },
        keyname: {
            private: true,
            default: function () { return uuid.v4(); },
            required: true
        },
    });

    mf.load = function (key, callback) {
        //if this isn't a child and the user didn't include the prefix, add it
        if (key.indexOf(mf.options.childsep) === -1 && key.indexOf(mf.options.prefix) !== 0)  {
            key = mf.options.prefix + (mf.options.sep || '!') + key;
        }
        mf.options.db.get(key, function (err, result) {
            var obj;
            if (!err) {
                obj = mf.create(result);
                obj.key = key;
                callback(err, obj);
            } else {
                callback(err);
            }
        });
    };
    
    mf.get = mf.load;

    mf.delete = function (key, callback) {
        mf.options.db.del(key, callback);
    };

    mf.update = function (key, updated_fields, callback) {
        mf.load(key, function (err, result) {
            if (!err && result) {
                var keys = Object.keys(updated_fields);
                for (var idx = 0; idx < keys.length; idx++) {
                    result[keys[idx]] = updated_fields[keys[idx]];
                }
                result.save(function (err) {
                    callback(err, result);
                });
            } else {
                callback(err);
            }
        });
    };

    mf.all = function (callback) {
        var objects = [];
        var err;
        var stream = mf.options.db.createReadStream({
            start : mf.options.prefix,
            end : mf.options.prefix + '~'
        });
        stream.on('data', function (entry) {
            if (entry.key.indexOf(mf.options.childsep || '~') == -1) {
                var inst = mf.create(entry.value);
                inst.key = entry.key;
                objects.push(inst);
            }
        });
        stream.on('error', function (error) {
            err = error;
            callback(err, null);
        });
        stream.on('close', function () {
            callback(err, objects);
        });
    };

    function indexName(factory, field, value) {
        var hash = crypto.createHash('md5').update(value).digest('hex');
        return '__index__' + (factory.options.sep || '!') + factory.options.prefix + (factory.options.sep || '!') + field + (factory.options.sep || '!') + hash;
    }


    mf.getByIndex = function (field, value, callback, _keyfilter, limit) {
        if (this.definition[field].hasOwnProperty('index') && this.definition[field].index === true) {
            mf.options.db.get(indexName(mf, field, value), function (err, index) {
                var keys;
                if (err || !index) index = {};
                if (index.hasOwnProperty(value)) {
                    keys = index[value];
                    if (_keyfilter) {
                        keys = keys.filter(function (key) {
                            return key.indexOf(_keyfilter) === 0;
                        });
                    }
                    if (limit) {
                        keys = keys.slice(0, limit);
                    }
                    async.map(keys, function (key, acb) {
                        mf.options.db.get(key, function (err, result) {
                            var obj;
                            if (!err) {
                                obj = mf.create(result);
                                obj.key = key;
                                acb(err, obj);
                            } else {
                                acb(err);
                            }
                        }.bind(this));
                    }.bind(this),
                    function (err, results) {
                        if (limit === 1) {
                            callback(err, results[0]);
                        } else {
                            callback(err, results);
                        }
                    });
                } else {
                    callback("no index for value");
                }
            }.bind(this));
        } else {
            callback("field does not have index");
        }
    };

    mf.findByIndex = function (field, value, callback, _keyfilter) {
        mf.getByIndex.call(this, field, value, callback, _keyfilter, 1);
    };

    mf.extendModel({
        save: function (callback) {
            async.waterfall([
                function (acb) {
                    this.__verymeta.db.put(this.key, this.toJSON(), acb);
                }.bind(this),
                function (acb) {
                    async.each(Object.keys(this.__verymeta.defs), function (field, scb) {
                        var ikey;
                        if (this.__verymeta.defs[field].hasOwnProperty('index') && this.__verymeta.defs[field].index === true) {
                            ikey = indexName(mf, field, this[field]);
                            this.__verymeta.db.get(ikey, function (err, obj) {
                                var objkeys, idx, kidx;
                                if (err || !obj) {
                                    obj = {};
                                }
                                objkeys = Object.keys(obj);
                                for (kidx in objkeys) {
                                    idx = obj[objkeys[kidx]].indexOf(this.key);
                                    if (idx !== -1) {
                                        obj[objkeys[kidx]].pop(idx);
                                    }
                                }
                                if (!obj.hasOwnProperty(this[field])) obj[this[field]] = [];
{'fraction_non_alphanumeric': 0.1263789746917586, 'fraction_numerical': 0.0018656716417910447, 'mean_word_length': 1.0696659392311567, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 22, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4948195', 'n_tokens_mistral': 3287, 'n_tokens_neox': 3099, 'n_words': 638}