<commit_before>var _ = require('lodash');
var moment = require('moment');
var Promise = require('bluebird');
var debug = require('debug')('lib:various');
var crypto = require('crypto');
var request = Promise.promisifyAll(require('request'));
var fs = Promise.promisifyAll(require('fs'));
var nconf = require('nconf');

var mongo = require('./mongo');

var Geo = require('node-geoip');
var G = new Geo.GeoIP(Geo.Database);

function getGeoIP(sourceIP) {
    var retVal = null;
    // TODO handle 10.x.x.x 127.x.x.x 192.168.x.x 172.16.x.x as "reserved" ?
    if(!_.isUndefined(sourceIP)) {
        try {
            retVal = G.getCountry(sourceIP);
            // debug("GeoIP of %s return %j", sourceIP, retVal);
        } catch (ex) {
            retVal = {'code': null, 'country': null, 'ip': sourceIP};
            // debug("GeoIP of %s %s", sourceIP, ex);
        }
    } else {
        retVal = {'code': null, 'country': null, 'ip': sourceIP};
        // debug("GeoIP absent for %s!", sourceIP);
    }
    return retVal;
};

function accessLog(funcName, request, computed) {
    var sourceIP = _.get(request.headers, "x-forwarded-for");
    var geoinfo = getGeoIP(sourceIP);
    var accessInfo = {
        when: moment().toISOString(),
        ccode: geoinfo.code,
        referer: request.headers.referer,
        details: request.url
    };
    if(!_.isUndefined(computed.error))
        accessInfo.error = computed.error;

    accessLogBuffer.push(accessInfo);
};

var accessLogBuffer = [];

function accessLogFlush() {
    /*
    var whenDet = _.map(accessLogBuffer, function(ace) {
        return moment.duration( moment() - moment(ace.when) ).humanize();
    });
    lib:various Flushing 20 collected accesses: ["a minute","a minute","a minute","a minute","a minute","a minute","a minute","a minute","a minute","a minute","a few seconds","a few seconds","a few seconds","a few seconds","a few seconds","a few seconds","a few seconds","a few seconds","a few seconds","a few seconds"]... +15ms
    debug("Flushing %d collected accesses: %j...",
        _.size(accessLogBuffer), whenDet);
    */
    debug("Flushing %d collected accesses", _.size(accessLogBuffer) );

    var javascriptLovableCopy = new Array();
    _.times( _.size(accessLogBuffer), function(i) {
        javascriptLovableCopy.push(accessLogBuffer.pop() );
    });

    return mongo
        .writeMany(nconf.get('schema').accesses, javascriptLovableCopy);
};

function hash(obj, fields) {
    if(_.isUndefined(fields))
        fields = _.keys(obj);
    var plaincnt = fields.reduce(function(memo, fname) {
        return memo += fname + "∴" + _.get(obj, fname, '…miss!') + ",";
        return memo;
    }, "");
    // debug("Hashing of %s", plaincnt);
    sha1sum = crypto.createHash('sha1');
    sha1sum.update(plaincnt);
    return sha1sum.digest('hex');
};

function loadJSONfile(fname) {
    debug("Opening file %s", fname);
    return fs
        .readFileAsync(fname, "utf-8")
        .then(JSON.parse)
        .tap(function(check) {
            if(!_.size(check))
                throw new Error("File " + fname + " #" + _.size(check));
        });
};

function loadJSONurl(url) {
    debug("opening url %s", url);
    return request
        .getAsync(url)
        .then(function(res) {
            return res.body;
        })
        .then(JSON.parse);
};

module.exports = {
    hash: hash,
    getGeoIP: getGeoIP,

    accessLog: accessLog,
    accessLogBuffer: accessLogBuffer,
    accessLogFlush: accessLogFlush,

    loadJSONurl: loadJSONurl,
    loadJSONfile: loadJSONfile
};<commit_msg>a change found in the production server: wtf<commit_after>var _ = require('lodash');
var moment = require('moment');
var Promise = require('bluebird');
var debug = require('debug')('lib:various');
var crypto = require('crypto');
var request = Promise.promisifyAll(require('request'));
var fs = Promise.promisifyAll(require('fs'));
var nconf = require('nconf');

var mongo = require('./mongo');

var Geo = require('node-geoip');
var G = new Geo.GeoIP(Geo.Database);

function getGeoIP(sourceIP) {
    var retVal = null;
    // TODO handle 10.x.x.x 127.x.x.x 192.168.x.x 172.16.x.x as "reserved" ?
    if(!_.isUndefined(sourceIP)) {
        try {
            retVal = G.getCountry(sourceIP);
            // debug("GeoIP of %s return %j", sourceIP, retVal);
        } catch (ex) {
            retVal = {'code': null, 'country': null, 'ip': sourceIP};
            // debug("GeoIP of %s %s", sourceIP, ex);
        }
    } else {
        retVal = {'code': null, 'country': null, 'ip': sourceIP};
        // debug("GeoIP absent for %s!", sourceIP);
    }
    return retVal;
};

function accessLog(funcName, request, computed) {
    var sourceIP = _.get(request.headers, "x-forwarded-for");
    var geoinfo = getGeoIP(sourceIP);
    var accessInfo = {
        when: moment().toISOString(),
        ccode: geoinfo.code,
        referer: request.headers.referer,
        details: request.url
    };
    if(!_.isUndefined(computed.error))
        accessInfo.error = computed.error;

    accessLogBuffer.push(accessInfo);
};

var accessLogBuffer = [];

function accessLogFlush() {
    /*
    var whenDet = _.map(accessLogBuffer, function(ace) {
        return moment.duration( moment() - moment(ace.when) ).humanize();
    });
    lib:various Flushing 20 collected accesses: ["a minute","a minute","a minute","a minute","a minute","a minute","a minute","a minute","a minute","a minute","a few seconds","a few seconds","a few seconds","a few seconds","a few seconds","a few seconds","a few seconds","a few seconds","a few seconds","a few seconds"]... +15ms
    debug("Flushing %d collected accesses: %j...",
        _.size(accessLogBuffer), whenDet);
    */
    debug("Flushing %d collected accesses", _.size(accessLogBuffer) );

    var javascriptLovableCopy = new Array();
    _.times( _.size(accessLogBuffer), function(i) {
        javascriptLovableCopy.push(accessLogBuffer.pop() );
    });

    return mongo
        .writeMany(nconf.get('schema').accesses, javascriptLovableCopy);
};

function hash(obj, fields) {
    if(_.isUndefined(fields))
        fields = _.keys(obj);
    var plaincnt = fields.reduce(function(memo, fname) {
        return memo += fname + "∴" + _.get(obj, fname, '…miss!') + ",";
        return memo;
    }, "");
    // debug("Hashing of %s", plaincnt);
    sha1sum = crypto.createHash('sha1');
    sha1sum.update(plaincnt);
    return sha1sum.digest('hex');
};

function loadJSONfile(fname) {
    debug("Opening file %s", fname);
    return fs
        .readFileAsync(fname, "utf-8")
        .then(JSON.parse)
        .tap(function(check) {
            if(!_.size(check))
                throw new Error("File " + fname + " #" + _.size(check));
        });
};

function loadJSONurl(url) {
    debug("opening url %s", url);
    return request
        .getAsync(url)
        .then(function(res) {
            return res.body;
        })
        .then(JSON.parse);
};

function dumpJSONfile(fname, content) {
    const stringified = JSON.stringify(content, undefined, 1);
    debug("Opening file %s for write there %d bytes", fname, _.size(stringified));
    return fs
        .writeFileAsync(fname, stringified, "utf-8");
};

module.exports = {
    hash: hash,
    getGeoIP: getGeoIP,

    accessLog: accessLog,
    accessLogBuffer: accessLogBuffer,
    accessLogFlush: accessLogFlush,

    loadJSONurl: loadJSONurl,
    loadJSONfile: loadJSONfile,
    dumpJSONfile: dumpJSONfile
};
{'fraction_non_alphanumeric': 0.16182572614107885, 'fraction_numerical': 0.0069602462856378, 'mean_word_length': 3.015045674368619, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1301306', 'n_tokens_mistral': 2453, 'n_tokens_neox': 2377, 'n_words': 552}