<commit_before>var _ = require('lodash');
var moment = require('moment');
var Promise = require('bluebird');
var mongodb = Promise.promisifyAll(require('mongodb'));
var debug = require('debug')('lib:mongo');
var debugread = require('debug')('lib:mongo:read');
var debugcount = require('debug')('lib:mongo:count');
var nconf = require('nconf');

var performa = require('./performa');

var dbConnection = function() {
    var url = nconf.get('mongodb');
    return mongodb
        .MongoClient
        .connectAsync(url)
        .disposer(function(db) {
            return db.close();
        });
};

var writeOne = function(cName, dataObject) {
    var p = performa.begin('writeOne',  cName, null);
    return Promise.using(dbConnection(), function(db) {
        return db
            .collection(cName)
            .insert(dataObject);
    })
    .tap(function() { return performa.complete(p); })
    .return(dataObject);
};

var updateOne = function(cName, selector, updated) {
    var p = performa.begin('updateOne',  cName, {
        'selector': selector
    });
    return Promise.using(dbConnection(), function(db) {
        return db
            .collection(cName)
            .updateOne(selector, updated);
    })
    .tap(function() { return performa.complete(p); })
    .return(updated);
};

/* better safe than sorry */
var upsertOne = function(cName, selector, updated) {
    var p = performa.begin('upsertOne',  cName, {
        'selector': selector
    });
    return Promise.using(dbConnection(), function(db) {
        return db
            .collection(cName)
            .updateOne(selector, updated, {upsert: true});
    })
    .tap(function() { return performa.complete(p); })
    .return(updated);
};

function writeMany(cName, dataObjects) {
    var p = performa.begin('writeMany',  cName, {
        'size': _.size(dataObjects)
    });
    return Promise.using(dbConnection(), function(db) {
        return db
            .collection(cName)
            .insertMany(dataObjects);
    })
    .tap(function() { return performa.complete(p); });
};

function read(cName, selector, sorter) {
    if(_.isUndefined(sorter)) sorter = {};
    var p = performa.begin('read',  cName, {
        'selector': selector,
        'sorter': sorter
    });
    return Promise.using(dbConnection(), function(db) {
        return db
            .collection(cName)
            .find(selector)
            .sort(sorter)
            .toArray();
    })
    .tap(function() { return performa.complete(p); })
    .tap(function(rv) {
        debugread("read in %s by %j sort by %j → read %d objects", 
            cName, selector, sorter, _.size(rv) );
    });
};

function remove(cName, selector) {

    if(_.size(_.keys(selector)) === 0)
        throw new Error("Nope, you can't delete {} ");

    var p = performa.begin('remove',  cName, {
        'selector': selector
    });

    debug("Removing documents %j from %s", selector, cName);
    return Promise.using(dbConnection(), function(db) {
        return db
            .collection(cName)
            .remove(selector);
    })
    .tap(function() { return performa.complete(p); });
};

var readLimit = function(cName, selector, sorter, limitN, past) {
    if(_.isNaN(past)) past = 0;
    var p = performa.begin('readLimit',  cName, {
        'selector': selector,
        'sorter': sorter,
        'limit': limitN,
        'skip': past
    });
    debug("readLimit in %s by %j limit %d skip %d", 
        cName, selector, limitN, past);
    return Promise.using(dbConnection(), function(db) {
        return db
            .collection(cName)
            .find(selector)
            .sort(sorter)
            .skip(past)
            .limit(limitN)
            .toArray()
    })
    .tap(function() { return performa.complete(p); })
    .tap(function(rv) {
        debug("readLimit by %j →  gives %d objects", selector, _.size(rv) );
    })
    .catch(function(errstr) {
        var alarms = require('./alarms');
        alarms.reportAlarm({
            caller: "readLimit",
            what: errstr,
            info: { selector: selector, limit: limitN, sorter: sorter }
        });
        debug("Error in readLimit!: %s", errstr);
        return [];
    });
};

var countByMatch = function(cName, selector) {
    var p = performa.begin('countByMatch',  cName, {
        'selector': selector
    });
    debugcount("countByMatch in %s by %j", cName, selector);
    return Promise.using(dbConnection(), function(db) {
        return db
            .collection(cName)
            .find(selector)
            .count();
    })
    .tap(function() { return performa.complete(p); });
};

var aggregate = function(cName, match, group) {
    var p = performa.begin('aggregate',  cName, {
        'selector': match,
        'group': group
    });
    return Promise.using(dbConnection(), function(db) {
        return db
            .collection(cName)
            .aggregate([
                { $match: match },
                { $group: group }
            ])
            .toArray();
    })
    .tap(function(ret) {
        debug("aggregate %s match %s group %s → %d entries",
            cName, JSON.stringify(match),
            JSON.stringify(group), _.size(ret));
    })
    .tap(function() { return performa.complete(p); });
};

/* This is *XXX FIXME* because the mongo query returns a number
 * of record equal to timeline, and this is bad. just I don't know
 * exactly how to keep only the entries with 'users' bigger than X,
 * so I'm doing the filtering with JS. This is used in 'staticpages'
 * and in 'postReality' */
var getPostRelations = function(cName, filter) {
    throw new Error("Unsupported ATM");

    return Promise.using(dbConnection(), function(db) {
        return db
            .collection(cName)
            .aggregate([
                { $match: filter },
                {
                    $group: {
                        _id: { "postId": "$postId"},
                        users: { $addToSet: "$userId"},
                        times: { $addToSet: "$displayTime"}
                    }
                }
            ])
            .toArray();
    })
    .tap(function() { return performa.complete(p); });
};

var countByDay = function(cName, timeVarName, filter, aggext) {

    if(!_.startsWith(timeVarName, '$'))
        throw new Error("developer please: mongoVar wants '$'");

    var p = performa.begin('countByDay',  cName, {
        'timeVariable': timeVarName,
        'selector': filter,
        'group': aggext
    });
    var queryId = { 
        year:  { $year: timeVarName },
        month: { $month: timeVarName },
        day:   { $dayOfMonth: timeVarName }
    };

    if(_.isObject(aggext) && _.size(_.keys(aggext)) > 0) {
        /* for example: { user: "$userId" } */
        queryId = _.extend(queryId, aggext);
    }

    var totalQ = [
        { $match: filter },
        { $group: {
            _id: queryId,
            count: { $sum: 1 }
        }}];

    debug("countByDay on %s %j  → ", cName, filter);
    return Promise.using(dbConnection(), function(db) {
        return db
            .collection(cName)
            .aggregate(totalQ)
            .toArray()
            .catch(function(error) {
                var alarms = require('./alarms');
                alarms.reportAlarm({
                    caller: "countByDay",
                    what: error,
                    info: { cName: cName,
                            timeVarName: timeVarName,
                            filter: filter,
                            aggext: aggext
                    }
                });
                debug("mongo error: %s (%s)", error, cName);
                return [];
            });
    })
    .tap(function() { return performa.complete(p); })
    .tap(function(done) {
        debug("← countByDay on %s F %j[%j] got %d",
            cName, filter, aggext, _.size(done));
    });
};

var countByObject = function(cName, idobj) {
    if(_.isUndefined(idobj)) idobj = {};
    debug("countByObject in %s by %j", cName, idobj);
    var p = performa.begin('countByObject',  cName, {
        'group': idobj
    });
    return Promise.using(dbConnection(), function(db) {
        return db
            .collection(cName)
            .aggregate([
                {
                  $group: {
                    _id: idobj,
                    count: { $sum: 1 }
                  }
                },
                { $sort: { count: -1 } }
            ])
            .toArray()
            .catch(function(error) {
                var alarms = require('./alarms');
                alarms.reportAlarm({
                    caller: "countByObject",
                    what: error,
                    info: { cName: cName, idobj: idobj }
                });
                debug("MongoQuery %s error: %s", cName, error);
                return [];
            });
    })
    .tap(function() { return performa.complete(p); });
};

function cacheFlush(cache, name) {

    debug("cacheFlush `%s` of %d elements", name, _.size(cache) );

    var javascriptLovableCopy = new Array();
    _.times( _.size(cache), function(i) {
        javascriptLovableCopy.push(cache.pop() );
    });

    return Promise.using(dbConnection(), function(db) {
        return db
            .collection(nconf.get('schema').performa)
            .insertMany(javascriptLovableCopy);
    });
};

function updateMany(cName, elist) {
    /* elist is supposed to have "_id" field */
    _.each(elist, function(e) {
        if(_.isUndefined(e['id'])) {
            debug("Error with elements %s", JSON.stringify(e, undefined, 2));
            throw new Error("we need id in every element");
        }
    });
    var p = performa.begin('updateMany', cName, {
        'size': _.size(elist)
    });
    debug("updateMany in %s with %d elements", cName, _.size(elist));
    return Promise.each(elist, function(e) {
        return Promise.using(dbConnection(), function(db) {
            return db
                .collection(cName)
                .update({'id': e.id}, e);
        });
    })
    .tap(function() { return performa.complete(p); });
};

function lookup(cName, query, sequence) {
    // debug("lookup on %s of %j + %j", cName, query, sequence);
    var p = performa.begin('lookup', cName, {
        query: query,
        sequence: sequence
    });

    return Promise.using(dbConnection(), function(db) {
        return db
            .collection(cName)
            .aggregate([ query, sequence ])
            .toArray()
            .catch(function(error) {
                var alarms = require('./alarms');
                alarms.reportAlarm({
                    caller: "lookup",
                    what: error,
                    info: { cName: cName, query: query, sequence: sequence }
                });
                debug("MongoQuery %s error: %s", cName, error);
                return [];
            });
    })
    .tap(function() { return performa.complete(p); });
};

function save(cName, doc) {
    return Promise.using(dbConnection(), function(db) {
        return db
            .collection(cName)
            .save(doc);
    });
};

function createIndex(cName, index, opt) {
    return Promise.using(dbConnection(), function(db) {
        return db.ensureIndex(cName, index, opt);
    })
    .tap(function(results) {
        debug("indexes created on %s: %j = %j", cName, index, results);
    });
};

function distinct(cName, field, query) {
    debug("distinct in %s for %s with %j", cName, field, query);
    return Promise.using(dbConnection(), function(db) {
        return db
            .collection(cName)
            .distinct(field, query);
    })
    .tap(function(results) {
        debug("distinct on %s, on %s: %d elements", cName, field, _.size(results));
    });
};

module.exports = {
    updateOne: updateOne,
    upsertOne: upsertOne,
    writeOne: writeOne,
    writeMany: writeMany,
    readLimit: readLimit,
    countByDay: countByDay,
    countByMatch: countByMatch,
    countByObject: countByObject,
    getPostRelations: getPostRelations,
    read: read,
    remove: remove,
    aggregate: aggregate,
    cacheFlush: cacheFlush,
    updateMany: updateMany,
    lookup: lookup,
    save: save,
    createIndex: createIndex,
    distinct: distinct
};
<commit_msg>small tuninf of the useless verbosity I'm use to<commit_after>var _ = require('lodash');
var moment = require('moment');
var Promise = require('bluebird');
var mongodb = Promise.promisifyAll(require('mongodb'));
var debug = require('debug')('lib:mongo');
var debugread = require('debug')('lib:mongo:read');
var debugcount = require('debug')('lib:mongo:count');
var nconf = require('nconf');

var performa = require('./performa');

var dbConnection = function() {
    var url = nconf.get('mongodb');
    return mongodb
        .MongoClient
        .connectAsync(url)
        .disposer(function(db) {
            return db.close();
        });
};

var writeOne = function(cName, dataObject) {
    var p = performa.begin('writeOne',  cName, null);
    return Promise.using(dbConnection(), function(db) {
        return db
            .collection(cName)
            .insert(dataObject);
    })
    .tap(function() { return performa.complete(p); })
    .return(dataObject);
};

var updateOne = function(cName, selector, updated) {
    var p = performa.begin('updateOne',  cName, {
        'selector': selector
    });
    return Promise.using(dbConnection(), function(db) {
        return db
            .collection(cName)
            .updateOne(selector, updated);
    })
    .tap(function() { return performa.complete(p); })
    .return(updated);
};

/* better safe than sorry */
var upsertOne = function(cName, selector, updated) {
    var p = performa.begin('upsertOne',  cName, {
        'selector': selector
    });
    return Promise.using(dbConnection(), function(db) {
        return db
            .collection(cName)
            .updateOne(selector, updated, {upsert: true});
    })
    .tap(function() { return performa.complete(p); })
    .return(updated);
};

function writeMany(cName, dataObjects) {
    var p = performa.begin('writeMany',  cName, {
        'size': _.size(dataObjects)
    });
    return Promise.using(dbConnection(), function(db) {
        return db
            .collection(cName)
            .insertMany(dataObjects);
    })
    .tap(function() { return performa.complete(p); });
};

function read(cName, selector, sorter) {
    if(_.isUndefined(sorter)) sorter = {};
    var p = performa.begin('read',  cName, {
        'selector': selector,
        'sorter': sorter
    });
    return Promise.using(dbConnection(), function(db) {
        return db
            .collection(cName)
            .find(selector)
            .sort(sorter)
            .toArray();
    })
    .tap(function() { return performa.complete(p); })
    .tap(function(rv) {
        debugread("read in %s by %j sort by %j → read %d objects", 
            cName, selector, sorter, _.size(rv) );
    });
};

function remove(cName, selector) {

    if(_.size(_.keys(selector)) === 0)
        throw new Error("Nope, you can't delete {} ");

    var p = performa.begin('remove',  cName, {
        'selector': selector
    });

    debug("Removing documents %j from %s", selector, cName);
    return Promise.using(dbConnection(), function(db) {
        return db
            .collection(cName)
            .remove(selector);
    })
    .tap(function() { return performa.complete(p); });
};

var readLimit = function(cName, selector, sorter, limitN, past) {
    if(_.isNaN(past)) past = 0;
    var p = performa.begin('readLimit',  cName, {
        'selector': selector,
        'sorter': sorter,
        'limit': limitN,
        'skip': past
    });
    // debug("readLimit in %s by %j limit %d skip %d", 
    //     cName, selector, limitN, past);
    return Promise.using(dbConnection(), function(db) {
        return db
            .collection(cName)
            .find(selector)
            .sort(sorter)
            .skip(past)
            .limit(limitN)
            .toArray()
    })
    .tap(function() { return performa.complete(p); })
    .tap(function(rv) {
        debug("readLimit by %j →  gives %d objects", selector, _.size(rv) );
    })
    .catch(function(errstr) {
        var alarms = require('./alarms');
        alarms.reportAlarm({
            caller: "readLimit",
            what: errstr,
            info: { selector: selector, limit: limitN, sorter: sorter }
        });
        debug("Error in readLimit!: %s", errstr);
        return [];
    });
};

var countByMatch = function(cName, selector) {
    var p = performa.begin('countByMatch',  cName, {
        'selector': selector
    });
    debugcount("countByMatch in %s by %j", cName, selector);
    return Promise.using(dbConnection(), function(db) {
        return db
            .collection(cName)
            .find(selector)
            .count();
    })
    .tap(function() { return performa.complete(p); });
};

var aggregate = function(cName, match, group) {
    var p = performa.begin('aggregate',  cName, {
        'selector': match,
        'group': group
    });
    return Promise.using(dbConnection(), function(db) {
        return db
            .collection(cName)
            .aggregate([
                { $match: match },
                { $group: group }
            ])
            .toArray();
    })
    .tap(function(ret) {
        debug("aggregate %s match %s group %s → %d entries",
            cName, JSON.stringify(match),
            JSON.stringify(group), _.size(ret));
    })
    .tap(function() { return performa.complete(p); });
};

/* This is *XXX FIXME* because the mongo query returns a number
 * of record equal to timeline, and this is bad. just I don't know
 * exactly how to keep only the entries with 'users' bigger than X,
 * so I'm doing the filtering with JS. This is used in 'staticpages'
 * and in 'postReality' */
var getPostRelations = function(cName, filter) {
    throw new Error("Unsupported ATM");

    return Promise.using(dbConnection(), function(db) {
        return db
            .collection(cName)
            .aggregate([
                { $match: filter },
                {
                    $group: {
                        _id: { "postId": "$postId"},
                        users: { $addToSet: "$userId"},
                        times: { $addToSet: "$displayTime"}
                    }
                }
            ])
            .toArray();
    })
    .tap(function() { return performa.complete(p); });
};

var countByDay = function(cName, timeVarName, filter, aggext) {

    if(!_.startsWith(timeVarName, '$'))
        throw new Error("developer please: mongoVar wants '$'");

    var p = performa.begin('countByDay',  cName, {
        'timeVariable': timeVarName,
        'selector': filter,
        'group': aggext
    });
    var queryId = { 
        year:  { $year: timeVarName },
        month: { $month: timeVarName },
        day:   { $dayOfMonth: timeVarName }
    };

    if(_.isObject(aggext) && _.size(_.keys(aggext)) > 0) {
        /* for example: { user: "$userId" } */
        queryId = _.extend(queryId, aggext);
    }

    var totalQ = [
        { $match: filter },
        { $group: {
            _id: queryId,
            count: { $sum: 1 }
        }}];

    debug("countByDay on %s %j  → ", cName, filter);
    return Promise.using(dbConnection(), function(db) {
        return db
            .collection(cName)
            .aggregate(totalQ)
            .toArray()
            .catch(function(error) {
                var alarms = require('./alarms');
                alarms.reportAlarm({
                    caller: "countByDay",
                    what: error,
                    info: { cName: cName,
                            timeVarName: timeVarName,
                            filter: filter,
                            aggext: aggext
                    }
                });
                debug("mongo error: %s (%s)", error, cName);
                return [];
            });
    })
    .tap(function() { return performa.complete(p); })
    .tap(function(done) {
        debug("← countByDay on %s F %j[%j] got %d",
            cName, filter, aggext, _.size(done));
    });
};

var countByObject = function(cName, idobj) {
    if(_.isUndefined(idobj)) idobj = {};
    debug("countByObject in %s by %j", cName, idobj);
    var p = performa.begin('countByObject',  cName, {
        'group': idobj
    });
    return Promise.using(dbConnection(), function(db) {
        return db
            .collection(cName)
            .aggregate([
                {
                  $group: {
                    _id: idobj,
                    count: { $sum: 1 }
                  }
                },
                { $sort: { count: -1 } }
            ])
            .toArray()
            .catch(function(error) {
                var alarms = require('./alarms');
                alarms.reportAlarm({
                    caller: "countByObject",
                    what: error,
                    info: { cName: cName, idobj: idobj }
                });
                debug("MongoQuery %s error: %s", cName, error);
                return [];
            });
    })
    .tap(function() { return performa.complete(p); });
};

function cacheFlush(cache, name) {

    debug("cacheFlush `%s` of %d elements", name, _.size(cache) );

    var javascriptLovableCopy = new Array();
    _.times( _.size(cache), function(i) {
        javascriptLovableCopy.push(cache.pop() );
    });

    return Promise.using(dbConnection(), function(db) {
        return db
            .collection(nconf.get('schema').performa)
            .insertMany(javascriptLovableCopy);
    });
};

function updateMany(cName, elist) {
    /* elist is supposed to have "_id" field */
    _.each(elist, function(e) {
        if(_.isUndefined(e['id'])) {
            debug("Error with elements %s", JSON.stringify(e, undefined, 2));
            throw new Error("we need id in every element");
        }
    });
    var p = performa.begin('updateMany', cName, {
        'size': _.size(elist)
    });
    debug("updateMany in %s with %d elements", cName, _.size(elist));
    return Promise.each(elist, function(e) {
        return Promise.using(dbConnection(), function(db) {
            return db
                .collection(cName)
                .update({'id': e.id}, e);
        });
    })
    .tap(function() { return performa.complete(p); });
};

function lookup(cName, query, sequence) {
    // debug("lookup on %s of %j + %j", cName, query, sequence);
    var p = performa.begin('lookup', cName, {
        query: query,
        sequence: sequence
    });

    return Promise.using(dbConnection(), function(db) {
        return db
            .collection(cName)
            .aggregate([ query, sequence ])
            .toArray()
            .catch(function(error) {
                var alarms = require('./alarms');
                alarms.reportAlarm({
                    caller: "lookup",
                    what: error,
                    info: { cName: cName, query: query, sequence: sequence }
                });
                debug("MongoQuery %s error: %s", cName, error);
                return [];
            });
    })
    .tap(function() { return performa.complete(p); });
};

function save(cName, doc) {
    return Promise.using(dbConnection(), function(db) {
        return db
            .collection(cName)
            .save(doc);
    });
};

function createIndex(cName, index, opt) {
    return Promise.using(dbConnection(), function(db) {
        return db.ensureIndex(cName, index, opt);
    })
    .tap(function(results) {
        debug("indexes created on %s: %j = %j", cName, index, results);
    });
};

function distinct(cName, field, query) {
    debug("distinct in %s for %s with %j", cName, field, query);
    return Promise.using(dbConnection(), function(db) {
        return db
            .collection(cName)
            .distinct(field, query);
    })
    .tap(function(results) {
        debug("distinct on %s, on %s: %d elements", cName, field, _.size(results));
    });
};

module.exports = {
    updateOne: updateOne,
    upsertOne: upsertOne,
    writeOne: writeOne,
    writeMany: writeMany,
    readLimit: readLimit,
    countByDay: countByDay,
    countByMatch: countByMatch,
    countByObject: countByObject,
    getPostRelations: getPostRelations,
    read: read,
    remove: remove,
    aggregate: aggregate,
    cacheFlush: cacheFlush,
    updateMany: updateMany,
    lookup: lookup,
    save: save,
    createIndex: createIndex,
    distinct: distinct
};

{'fraction_non_alphanumeric': 0.14556011206301514, 'fraction_numerical': 0.0005684355840675626, 'mean_word_length': 1.9635422933461677, 'pattern_counts': {'":': 2, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1589452', 'n_tokens_mistral': 7436, 'n_tokens_neox': 7226, 'n_words': 1708}