<commit_before>var _ = require('lodash');

// Removes old monitoring data. We only want basic monitoring with the last 100 events. 
// We keep last 80 and remove the rest to be sure.
function serverMonitoringCleanup(db, conn) {
    var exclude = { 
        eventDate: 0,
        pid: 0,
        version: 0,
        uptime: 0,
        network: 0,
        connectionName: 0,
        connections: 0,
        memory: 0,
        dataRetrieved: 0,
        docCounts: 0
    }

    var retainedRecords = (24 * 60) * 60 / 30; // 24 hours worth of 30 sec blocks (data refresh interval)

    db.find({connectionName: conn}).skip(retainedRecords).sort({eventDate: -1}).projection(exclude).exec(function (err, serverEvents) {
        var idArray = [];
        _.each(serverEvents, function(value, key) {
            idArray.push(value._id);
        });

        db.remove({'_id':{'$in': idArray}}, { multi: true }, function (err, newDoc) {});
    });
};

// runs a regular job against the connections and inserts into a local DB
var currDocCounts = {
    queried: 0,
    inserted: 0,
    deleted: 0,
    updated: 0
}

exports.serverMonitoring = function (monitoringDB, dbs) {
    if(dbs){
        Object.keys(dbs).forEach(function(key) {
            var adminDb = dbs[key].native.admin();
            adminDb.serverStatus(function(err, info) {   

                // if we got data back from db. If not, normally related to permissions
                var dataRetrieved = false;
                if(info){
                    dataRetrieved = true;
                }

                // doc numbers. We get the last interval number and subtract the current to get the diff
                var docCounts = getDocCounts(currDocCounts, info.metrics.document)

                var doc = {
                    eventDate: new Date(),
                    pid: info.pid,
                    version: info.version,
                    uptime: info.uptime,
                    activeClients: info.globalLock.activeClients,
                    connectionName: key,
                    connections: info.connections,
                    memory: info.mem,
                    dataRetrieved: dataRetrieved,
                    docCounts: docCounts
                }

                // insert the data into local DB
                monitoringDB.insert(doc, function (err, newDoc) {});

                // clean up old docs
                serverMonitoringCleanup(monitoringDB, key);
            });
        });
    }
}

function getDocCounts(currCounts, newCounts){
    var newDocCounts = {
        queried: 0,
        inserted: 0,
        deleted: 0,
        updated: 0
    }

    // queried
    if(currCounts.queried == 0){
        currCounts.queried = newCounts.returned;
    }else{
        newDocCounts.queried = newCounts.returned - currCounts.queried;
        currCounts.queried = newCounts.returned;
    }

    // inserts
    if(currCounts.inserted == 0){
        currCounts.inserted = newCounts.inserted;
    }else{
        newDocCounts.inserted = newCounts.inserted - currCounts.inserted;
        currCounts.inserted = newCounts.inserted;
    }

    // deleted
    if(currCounts.deleted == 0){
        currCounts.deleted = newCounts.deleted;
    }else{
        newDocCounts.deleted = newCounts.deleted - currCounts.deleted;
        currCounts.deleted = newCounts.deleted;
    }

    // updated
    if(currCounts.updated == 0){
        currCounts.updated = newCounts.updated;
    }else{
        newDocCounts.updated = newCounts.updated - currCounts.updated;
        currCounts.updated = newCounts.updated;
    }

    return newDocCounts;
}<commit_msg>Fix for issues monitoring connections which don't have admin access to DB<commit_after>var _ = require('lodash');

// Removes old monitoring data. We only want basic monitoring with the last 100 events. 
// We keep last 80 and remove the rest to be sure.
function serverMonitoringCleanup(db, conn) {
    var exclude = { 
        eventDate: 0,
        pid: 0,
        version: 0,
        uptime: 0,
        network: 0,
        connectionName: 0,
        connections: 0,
        memory: 0,
        dataRetrieved: 0,
        docCounts: 0
    }

    var retainedRecords = (24 * 60) * 60 / 30; // 24 hours worth of 30 sec blocks (data refresh interval)

    db.find({connectionName: conn}).skip(retainedRecords).sort({eventDate: -1}).projection(exclude).exec(function (err, serverEvents) {
        var idArray = [];
        _.each(serverEvents, function(value, key) {
            idArray.push(value._id);
        });

        db.remove({'_id':{'$in': idArray}}, { multi: true }, function (err, newDoc) {});
    });
};

// runs a regular job against the connections and inserts into a local DB
var currDocCounts = {
    queried: 0,
    inserted: 0,
    deleted: 0,
    updated: 0
}

exports.serverMonitoring = function (monitoringDB, dbs) {
    if(dbs){
        Object.keys(dbs).forEach(function(key) {
            var adminDb = dbs[key].native.admin();
            adminDb.serverStatus(function(err, info) {   

                // if we got data back from db. If not, normally related to permissions
                var dataRetrieved = false;
                if(info){
                    dataRetrieved = true;
                }

                // doc numbers. We get the last interval number and subtract the current to get the diff
                var docCounts = "";
                var activeClients = "";
                var pid = "N/A";
                var version = "N/A";
                var uptime = "N/A";
                var connections = "";
                var memory = "";

                // set the values if we can get them
                if(info){
                    docCounts = getDocCounts(currDocCounts, info.metrics.document);
                    activeClients = info.globalLock.activeClients;
                    pid = info.pid;
                    version = info.version;
                    uptime = info.uptime;
                    connections= info.connections;
                    memory = info.mem;
                }

                var doc = {
                    eventDate: new Date(),
                    pid: pid,
                    version: version,
                    uptime: uptime,
                    activeClients: activeClients,
                    connectionName: key,
                    connections: connections,
                    memory: memory,
                    dataRetrieved: dataRetrieved,
                    docCounts: docCounts
                }

                // insert the data into local DB
                monitoringDB.insert(doc, function (err, newDoc) {});

                // clean up old docs
                serverMonitoringCleanup(monitoringDB, key);
            });
        });
    }
}

function getDocCounts(currCounts, newCounts){
    var newDocCounts = {
        queried: 0,
        inserted: 0,
        deleted: 0,
        updated: 0
    }

    // queried
    if(currCounts.queried == 0){
        currCounts.queried = newCounts.returned;
    }else{
        newDocCounts.queried = newCounts.returned - currCounts.queried;
        currCounts.queried = newCounts.returned;
    }

    // inserts
    if(currCounts.inserted == 0){
        currCounts.inserted = newCounts.inserted;
    }else{
        newDocCounts.inserted = newCounts.inserted - currCounts.inserted;
        currCounts.inserted = newCounts.inserted;
    }

    // deleted
    if(currCounts.deleted == 0){
        currCounts.deleted = newCounts.deleted;
    }else{
        newDocCounts.deleted = newCounts.deleted - currCounts.deleted;
        currCounts.deleted = newCounts.deleted;
    }

    // updated
    if(currCounts.updated == 0){
        currCounts.updated = newCounts.updated;
    }else{
        newDocCounts.updated = newCounts.updated - currCounts.updated;
        currCounts.updated = newCounts.updated;
    }

    return newDocCounts;
}
{'fraction_non_alphanumeric': 0.09844559585492228, 'fraction_numerical': 0.010109945659042083, 'mean_word_length': 1.701946056674633, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5060394', 'n_tokens_mistral': 2368, 'n_tokens_neox': 2203, 'n_words': 562}