<commit_before>var _ = require('underscore');
var Q = require('q');
var moment = require('moment');
var crypto = require('crypto');
var mongoose = require('mongoose');
var mongo = mongoose.mongo;
var GridStore = mongo.GridStore;
var Schema = mongoose.Schema;
var ObjectId = mongoose.Types.ObjectId;
require('mongoose-long')(mongoose);

module.exports = function (dbUrl, injection) {
    var service = {};
    var db;
    var onConnectListeners = [];
    var models = {};

    var connection = mongoose.createConnection(dbUrl);

    connection.on('connected', function () {
        db = connection.db;
        onConnectListeners.map(function (listener) { return function () { return listener() }}).reduce(Q.when, Q(null));
    });

    function toPromise(query, method) {
        var deferred = Q.defer();
        query[method](function (err, result) {
            if (err) {
                deferred.reject(new Error(err));
            } else {
                deferred.resolve(result);
            }
        });
        return deferred.promise;
    }

    function deferredCallback(deferred) {
        return function (err, result) {
            if (err) {
                deferred.reject(new Error(err));
            } else {
                deferred.resolve(result);
            }
        }
    }

    service.addOnConnectListener = function (listener) {
        if (db) {
            listener();
        } else {
            onConnectListeners.push(listener);
        }
    };

    service.mongooseConnection = function () {
        return connection;
    };

    function modelFor(table) {
        if (models[table.entityTypeId]) {
            return models[table.entityTypeId];
        }
        var definition = _.chain(getAllFields(table)).map(function (field, fieldName) {
            if (field.fieldType.notPersisted) {
                return undefined;
            }
            var fieldType;
            if (field.fieldType.id === 'date') {
                fieldType = Date;
            } else if (field.fieldType.id === 'reference') {
                fieldType = {id: Schema.ObjectId, name: String};
            } else if (field.fieldType.id === 'attachment') {
                fieldType = {fileId: Schema.ObjectId, name: String};
            } else if (field.fieldType.id === 'integer') {
                fieldType = Number;
            } else if (field.fieldType.id === 'money') {
                fieldType = Schema.Types.Long;
            } else if (field.fieldType.id === 'checkbox') {
                if (field.fieldType.storeAsArrayField) {
                    return [field.fieldType.storeAsArrayField, [String]];
                } else {
                    fieldType = Boolean;
                }
            } else {
                fieldType = String;
            }
            if (field.isUnique) {
                fieldType = {type: fieldType, index: { unique: true }};
            }
            return [fieldName, fieldType];
        }).filter(_.identity).object().value();
        definition.__textIndex = [String];
        var schema = new Schema(definition, {
            collection: table.tableName
        });
        models[table.entityTypeId] = connection.model(table.entityTypeId, schema);
        return models[table.entityTypeId];
    }

    service.setupModel = function (table) {
        modelFor(table);
    };

    service.findCount = function (table, filteringAndSorting) {
        return onlyFilteringEnsureIndexes(table, filteringAndSorting).then(function () {
            return Q(modelFor(table).count(queryFor(table, filteringAndSorting)).exec());
        })
    };

    service.findAll = function (table, filteringAndSorting) {
        return ensureIndexes(table, filteringAndSorting).then(function () {
            return Q(modelFor(table).find(queryFor(table, filteringAndSorting)).sort(sortingFor(filteringAndSorting, table.fields)).exec())
                .then(function (result) { return result.map(fromBson(table.fields)) });
        });
    };

    service.findRange = function (table, filteringAndSorting, start, count) {
        return ensureIndexes(table, filteringAndSorting).then(function () {
            return Q(modelFor(table).find(queryFor(table, filteringAndSorting)).sort(sortingFor(filteringAndSorting, table.fields)).limit(count).skip(start).exec())
                .then(function (result) { return result.map(fromBson(table.fields)) });
        });
    };

    service.checkUserPassword = function (table, entityId, passwordField, password) {
        if (!entityId) {
            throw new Error('entityId should be defined for checkUserPassword()');
        }
        return Q(modelFor(table).findById(toMongoId(entityId)).exec()).then(function (user) {
            if (!user) {
                return false;
            }
            var userId = user._id.toString();
            var digest = service.passwordHash(userId, password);
            return digest === user.passwordHash ? fromBson(table.fields)(user) : false;
        });
    };

    function sortingFor(filteringAndSorting, fields) {
        return _.object(_.union(filteringAndSorting && filteringAndSorting.sorting && filteringAndSorting.sorting.filter(function (i) { return !!fields[i[0]]}) || [], [['modifyTime', -1]]));
    }

    function ensureIndexes(table, filteringAndSorting) {
        var indexFieldHash = {};
        var indexes = [];

        indexes = _.map(sortingFor(filteringAndSorting, table.fields), function (order, fieldName) { return [fieldName, order]});
        _.forEach(indexes, function (i) { indexFieldHash[i[0]] = true });

        return ensureIndexesWith(indexes, indexFieldHash, table, filteringAndSorting);
    }

    function onlyFilteringEnsureIndexes(table, filteringAndSorting) {
        return ensureIndexesWith([], {}, table, filteringAndSorting);
    }

    function ensureIndexesWith(indexes, indexFieldHash, table, filteringAndSorting) {
        var query = queryFor(table, filteringAndSorting);
        function addIndexField(q, fieldName) {
            if (fieldName === '$and' || fieldName === '$or') {
                _.forEach(q, function (qObj) {_.forEach(qObj, addIndexField) });
            } else if (!indexFieldHash[fieldName]) {
                indexes.unshift([fieldName, 1]);
                indexFieldHash[fieldName] = true;
            }
        }
        _.forEach(query, addIndexField);

        if (indexes.length > 0) {
            var collection = db.collection(table.tableName);
            return Q.nfbind(collection.ensureIndex.bind(collection))(indexes);
        } else {
            return Q(null);
        }
    }

    var systemFields = {
        createTime: {
            fieldType: {
                id: 'date'
            }
        },
        modifyTime: {
            fieldType: {
                id: 'date'
            }
        }
    };

    function getAllFields(table) {
        return _.extend({}, table.fields, systemFields);
    }

    function queryFor(table, filteringAndSorting) {
        var query = {};
        if (filteringAndSorting) {
            if (filteringAndSorting.query) {
                query = _.clone(filteringAndSorting.query); //TODO merging filtering and query
            }
            if (filteringAndSorting.textSearch) {
                var split = splitText(filteringAndSorting.textSearch);
                if (split.length > 0) {
                    query.$and = split.map(function (value) { return {__textIndex: { $regex: value + ".*" }}});
                }
            }
            if (filteringAndSorting.filtering) {
                var allFields = getAllFields(table);
                _.each(allFields, function (field, fieldName) {
                    if (_.isUndefined(filteringAndSorting.filtering[fieldName])) {
                        return;
                    }
                    var filterValue = filteringAndSorting.filtering[fieldName];
                    if (field.fieldType.id == 'reference') {
                        query[fieldName + '.id'] = toMongoId(filteringAndSorting.filtering[fieldName])
                    } else if (field.fieldType.id == 'checkbox') {
                        query[fieldName] = filteringAndSorting.filtering[fieldName] ? filteringAndSorting.filtering[fieldName] : {$in: [false, null]};
                    } else if (field.fieldType.id == 'date') {
                        if (filterValue.op === 'gt') {
                            query[fieldName] = {$gt: filterValue.value}; //TODO convert from string?
                        } else if (filterValue.op === 'lt') {
                            query[fieldName] = {$lt: filterValue.value}; //TODO convert from string?
                        } else {
                            query[fieldName] = filterValue;
                        }
                    } else if (field.fieldType.id == 'text') {
                        if (filterValue.op === 'startsWith') {
                            query[fieldName] = { $regex: filterValue.value + ".*" };
                        } else if (filterValue.op === 'in') {
                            query[fieldName] = { $in: filterValue.value };
                        } else {
                            query[fieldName] = filterValue;
                        }
                    } else {
                        query[fieldName] = filteringAndSorting.filtering[fieldName];
                    }
                });
            }
        }
        return query;
    }

    service.queryFor = queryFor;

    service.newEntityId = function () {
        return (new ObjectId()).toString();
    };

    service.createEntity = function (table, entity) {
        return callBeforeCrudListeners(table, null, entity).then(function () {
            var objectID = entity.id && toMongoId(entity.id) || new ObjectId();
            entity.id = (objectID).toString();
            var toInsert = toBson(table.fields)(entity);
            toInsert._id = objectID;
            toInsert.createTime = new Date();
            toInsert.modifyTime = new Date();
            setAuxiliaryFields(table.fields, toInsert, toInsert);
            return Q(modelFor(table).create(toInsert))
                .then(callAfterCrudListeners(table, null, fromBson(table.fields)(toInsert)))
                .then(function (result) {
                    return result._id;
                });
        });

    };

    service.aggregateQuery = function (table, aggregatePipeline) {
        var collection = db.collection(table.tableName);
        return Q.nfbind(collection.aggregate.bind(collection))(aggregatePipeline).then(function (rows) {
            return rows.map(fromBson(table.fields));
        })
    };

    function padId(id) {
        return id.length < 12 ? _.range(0, 12 - id.length).map(function () {return " "}).join("") + id : id;
    }

    function toMongoId(entityId) {
        return new ObjectId(padId(entityId));
    }

    service.readEntity = function (table, entityId) {
        if (!entityId) {
            throw new Error('entityId should be defined for readEntity()');
        }
        return Q(modelFor(table).findById(toMongoId(entityId)).exec()).then(function (result) {
            return fromBson(table.fields)(result);
        });
    };

    service.updateEntity = function (table, entity) {
        return service.readEntity(table, entity.id).then(function (oldEntity) {
            var newEntity = _.extend(Object.create(oldEntity), entity);
            return callBeforeCrudListeners(table, oldEntity, newEntity).then(function () {<commit_msg>Fix internal server error when access token is correct but user doesn't exist<commit_after>var _ = require('underscore');
var Q = require('q');
var moment = require('moment');
var crypto = require('crypto');
var mongoose = require('mongoose');
var mongo = mongoose.mongo;
var GridStore = mongo.GridStore;
var Schema = mongoose.Schema;
var ObjectId = mongoose.Types.ObjectId;
require('mongoose-long')(mongoose);

module.exports = function (dbUrl, injection) {
    var service = {};
    var db;
    var onConnectListeners = [];
    var models = {};

    var connection = mongoose.createConnection(dbUrl);

    connection.on('connected', function () {
        db = connection.db;
        onConnectListeners.map(function (listener) { return function () { return listener() }}).reduce(Q.when, Q(null));
    });

    function toPromise(query, method) {
        var deferred = Q.defer();
        query[method](function (err, result) {
            if (err) {
                deferred.reject(new Error(err));
            } else {
                deferred.resolve(result);
            }
        });
        return deferred.promise;
    }

    function deferredCallback(deferred) {
        return function (err, result) {
            if (err) {
                deferred.reject(new Error(err));
            } else {
                deferred.resolve(result);
            }
        }
    }

    service.addOnConnectListener = function (listener) {
        if (db) {
            listener();
        } else {
            onConnectListeners.push(listener);
        }
    };

    service.mongooseConnection = function () {
        return connection;
    };

    function modelFor(table) {
        if (models[table.entityTypeId]) {
            return models[table.entityTypeId];
        }
        var definition = _.chain(getAllFields(table)).map(function (field, fieldName) {
            if (field.fieldType.notPersisted) {
                return undefined;
            }
            var fieldType;
            if (field.fieldType.id === 'date') {
                fieldType = Date;
            } else if (field.fieldType.id === 'reference') {
                fieldType = {id: Schema.ObjectId, name: String};
            } else if (field.fieldType.id === 'attachment') {
                fieldType = {fileId: Schema.ObjectId, name: String};
            } else if (field.fieldType.id === 'integer') {
                fieldType = Number;
            } else if (field.fieldType.id === 'money') {
                fieldType = Schema.Types.Long;
            } else if (field.fieldType.id === 'checkbox') {
                if (field.fieldType.storeAsArrayField) {
                    return [field.fieldType.storeAsArrayField, [String]];
                } else {
                    fieldType = Boolean;
                }
            } else {
                fieldType = String;
            }
            if (field.isUnique) {
                fieldType = {type: fieldType, index: { unique: true }};
            }
            return [fieldName, fieldType];
        }).filter(_.identity).object().value();
        definition.__textIndex = [String];
        var schema = new Schema(definition, {
            collection: table.tableName
        });
        models[table.entityTypeId] = connection.model(table.entityTypeId, schema);
        return models[table.entityTypeId];
    }

    service.setupModel = function (table) {
        modelFor(table);
    };

    service.findCount = function (table, filteringAndSorting) {
        return onlyFilteringEnsureIndexes(table, filteringAndSorting).then(function () {
            return Q(modelFor(table).count(queryFor(table, filteringAndSorting)).exec());
        })
    };

    service.findAll = function (table, filteringAndSorting) {
        return ensureIndexes(table, filteringAndSorting).then(function () {
            return Q(modelFor(table).find(queryFor(table, filteringAndSorting)).sort(sortingFor(filteringAndSorting, table.fields)).exec())
                .then(function (result) { return result.map(fromBson(table.fields)) });
        });
    };

    service.findRange = function (table, filteringAndSorting, start, count) {
        return ensureIndexes(table, filteringAndSorting).then(function () {
            return Q(modelFor(table).find(queryFor(table, filteringAndSorting)).sort(sortingFor(filteringAndSorting, table.fields)).limit(count).skip(start).exec())
                .then(function (result) { return result.map(fromBson(table.fields)) });
        });
    };

    service.checkUserPassword = function (table, entityId, passwordField, password) {
        if (!entityId) {
            throw new Error('entityId should be defined for checkUserPassword()');
        }
        return Q(modelFor(table).findById(toMongoId(entityId)).exec()).then(function (user) {
            if (!user) {
                return false;
            }
            var userId = user._id.toString();
            var digest = service.passwordHash(userId, password);
            return digest === user.passwordHash ? fromBson(table.fields)(user) : false;
        });
    };

    function sortingFor(filteringAndSorting, fields) {
        return _.object(_.union(filteringAndSorting && filteringAndSorting.sorting && filteringAndSorting.sorting.filter(function (i) { return !!fields[i[0]]}) || [], [['modifyTime', -1]]));
    }

    function ensureIndexes(table, filteringAndSorting) {
        var indexFieldHash = {};
        var indexes = [];

        indexes = _.map(sortingFor(filteringAndSorting, table.fields), function (order, fieldName) { return [fieldName, order]});
        _.forEach(indexes, function (i) { indexFieldHash[i[0]] = true });

        return ensureIndexesWith(indexes, indexFieldHash, table, filteringAndSorting);
    }

    function onlyFilteringEnsureIndexes(table, filteringAndSorting) {
        return ensureIndexesWith([], {}, table, filteringAndSorting);
    }

    function ensureIndexesWith(indexes, indexFieldHash, table, filteringAndSorting) {
        var query = queryFor(table, filteringAndSorting);
        function addIndexField(q, fieldName) {
            if (fieldName === '$and' || fieldName === '$or') {
                _.forEach(q, function (qObj) {_.forEach(qObj, addIndexField) });
            } else if (!indexFieldHash[fieldName]) {
                indexes.unshift([fieldName, 1]);
                indexFieldHash[fieldName] = true;
            }
        }
        _.forEach(query, addIndexField);

        if (indexes.length > 0) {
            var collection = db.collection(table.tableName);
            return Q.nfbind(collection.ensureIndex.bind(collection))(indexes);
        } else {
            return Q(null);
        }
    }

    var systemFields = {
        createTime: {
            fieldType: {
                id: 'date'
            }
        },
        modifyTime: {
            fieldType: {
                id: 'date'
            }
        }
    };

    function getAllFields(table) {
        return _.extend({}, table.fields, systemFields);
    }

    function queryFor(table, filteringAndSorting) {
        var query = {};
        if (filteringAndSorting) {
            if (filteringAndSorting.query) {
                query = _.clone(filteringAndSorting.query); //TODO merging filtering and query
            }
            if (filteringAndSorting.textSearch) {
                var split = splitText(filteringAndSorting.textSearch);
                if (split.length > 0) {
                    query.$and = split.map(function (value) { return {__textIndex: { $regex: value + ".*" }}});
                }
            }
            if (filteringAndSorting.filtering) {
                var allFields = getAllFields(table);
                _.each(allFields, function (field, fieldName) {
                    if (_.isUndefined(filteringAndSorting.filtering[fieldName])) {
                        return;
                    }
                    var filterValue = filteringAndSorting.filtering[fieldName];
                    if (field.fieldType.id == 'reference') {
                        query[fieldName + '.id'] = toMongoId(filteringAndSorting.filtering[fieldName])
                    } else if (field.fieldType.id == 'checkbox') {
                        query[fieldName] = filteringAndSorting.filtering[fieldName] ? filteringAndSorting.filtering[fieldName] : {$in: [false, null]};
                    } else if (field.fieldType.id == 'date') {
                        if (filterValue.op === 'gt') {
                            query[fieldName] = {$gt: filterValue.value}; //TODO convert from string?
                        } else if (filterValue.op === 'lt') {
                            query[fieldName] = {$lt: filterValue.value}; //TODO convert from string?
                        } else {
                            query[fieldName] = filterValue;
                        }
                    } else if (field.fieldType.id == 'text') {
                        if (filterValue.op === 'startsWith') {
                            query[fieldName] = { $regex: filterValue.value + ".*" };
                        } else if (filterValue.op === 'in') {
                            query[fieldName] = { $in: filterValue.value };
                        } else {
                            query[fieldName] = filterValue;
                        }
                    } else {
                        query[fieldName] = filteringAndSorting.filtering[fieldName];
                    }
                });
            }
        }
        return query;
    }

    service.queryFor = queryFor;

    service.newEntityId = function () {
        return (new ObjectId()).toString();
    };

    service.createEntity = function (table, entity) {
        return callBeforeCrudListeners(table, null, entity).then(function () {
            var objectID = entity.id && toMongoId(entity.id) || new ObjectId();
            entity.id = (objectID).toString();
            var toInsert = toBson(table.fields)(entity);
            toInsert._id = objectID;
            toInsert.createTime = new Date();
            toInsert.modifyTime = new Date();
            setAuxiliaryFields(table.fields, toInsert, toInsert);
            return Q(modelFor(table).create(toInsert))
                .then(callAfterCrudListeners(table, null, fromBson(table.fields)(toInsert)))
                .then(function (result) {
                    return result._id;
                });
        });

    };

    service.aggregateQuery = function (table, aggregatePipeline) {
        var collection = db.collection(table.tableName);
        return Q.nfbind(collection.aggregate.bind(collection))(aggregatePipeline).then(function (rows) {
            return rows.map(fromBson(table.fields));
        })
    };

    function padId(id) {
        return id.length < 12 ? _.range(0, 12 - id.length).map(function () {return " "}).join("") + id : id;
    }

    function toMongoId(entityId) {
        return new ObjectId(padId(entityId));
    }

    service.readEntity = function (table, entityId) {
        if (!entityId) {
            throw new Error('entityId should be defined for readEntity()');
        }
        return Q(modelFor(table).findById(toMongoId(entityId)).exec()).then(function (result) {
            return result && fromBson(table.fields)(result) || result;
        });
    };

    service.updateEntity = function (table, entity) {
        return service.readEntity(table, entity.id).then(function (oldEntity) {
            var newEntity = _.extend(Object.create(oldEntity), entity);
            return callBeforeCrudListeners(table, oldEntity, newEntity).then(function () {
{'fraction_non_alphanumeric': 0.12388807323603075, 'fraction_numerical': 0.0009499956818378099, 'mean_word_length': 1.9588603551807844, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1974525', 'n_tokens_mistral': 6316, 'n_tokens_neox': 6094, 'n_words': 1206}