<commit_before>var _ = require('lodash');
var Promise = require('bluebird');
var fs = Promise.promisifyAll(require('fs'));
var request = Promise.promisifyAll(require('request'));
var cheerio = require('cheerio');
var debug = require('debug')('parser:⊹core');
var moment = require('moment');
var nconf = require('nconf'); 

nconf.argv().env();

function composeURL(what) {
    return [
        (nconf.get('url') || 'https://facebook.tracking.exposed' ),
        'api', 'v1', 'snippet', what
    ].join('/');
};

function snippetAvailable(config, what) {
    var url = composeURL(what);
    var requestpayload = {
        "since": nconf.get('since') ? nconf.get('since') : config.since,
        "until": nconf.get('until') ? nconf.get('until') : config.until,
        "parserName": config.name,
        "requirements": config.requirements || {}
    };

    /* id overwrites every other requirement */
    if(nconf.get('id'))
        requestpayload.requirements = { id : nconf.get('id') };

    debug("Connecting to %s", url);
    debug("⭐ %s", JSON.stringify(requestpayload, undefined, 2));

    var begin = moment();
    return request
        .postAsync(url, {form: requestpayload})
        .then(function(response) {
            if(_.size(response.body) < 3 && response.statusCode === 200) {
                debug("This query return zero HTMLs");
                return [];
            }
            return JSON.parse(response.body);
        })
        .catch(function(error) {
            debug("Error with %s: %s", url, error);
            throw new Error(error);
        })
        .tap(function() {
            debug("Execution completed in %d seconds",
                moment.duration(moment() - begin).asSeconds());
        });
};

function commitResult(config, newmeta, snippet) {
    debug("html [%s] [%s] +[%s]",
        moment.duration(moment(snippet.savingTime) - moment() ).humanize(true),
        _.keys(
            _.omit(snippet, [
                '_id', 'savingTime', 'id', 'userId',<commit_msg>improved coherence of mongo-script interface<commit_after>var _ = require('lodash');
var Promise = require('bluebird');
var fs = Promise.promisifyAll(require('fs'));
var request = Promise.promisifyAll(require('request'));
var cheerio = require('cheerio');
var debug = require('debug')('parser:⊹core');
var moment = require('moment');
var nconf = require('nconf'); 

nconf.argv().env();

function composeURL(what) {
    return [
        (nconf.get('url') || 'https://facebook.tracking.exposed' ),
        'api', 'v1', 'snippet', what
    ].join('/');
};

function snippetAvailable(config, what) {
    var url = composeURL(what);
    var requestpayload = {
        "since": moment().subtract(1, 'h').toISOString(),
        "until": moment().toISOString(),
        "parserName": config.name,
        "requirements": config.requirements || {}
    };

    /* if since or until are specify, use the command, 
     * otherwise use keep the default: last hour */
    if( nconf.get('since') || nconf.get('until') ) {
        debug("Remind: if you specify only one 'since' or 'until', the default is from the config");
        requestpayload.since = nconf.get('since') ? nconf.get('since') : config.since;
        requestpayload.until = nconf.get('until') ? nconf.get('until') : config.until;
    }

    /* id overwrites every other requirement */
    if(nconf.get('id')) {
        debug("Remind: when id is specify is not used the last hour. but $env 'since' 'until' or the default from the config");
        requestpayload.since = nconf.get('since') ? nconf.get('since') : config.since;
        requestpayload.until = nconf.get('until') ? nconf.get('until') : config.until;
        requestpayload.requirements = { id : nconf.get('id') };
    }

    debug("Connecting to %s", url);
    debug("⭐ %s", JSON.stringify(requestpayload, undefined, 2));

    var begin = moment();
    return request
        .postAsync(url, {form: requestpayload})
        .then(function(response) {
            if(_.size(response.body) < 3 && response.statusCode === 200) {
                debug("This query return zero HTMLs");
                return [];
            }
            return JSON.parse(response.body);
        })
        .catch(function(error) {
            debug("Error with %s: %s", url, error);
            throw new Error(error);
        })
        .tap(function() {
            debug("Execution completed in %d seconds",
                moment.duration(moment() - begin).asSeconds());
        });
};

function commitResult(config, newmeta, snippet) {
    debug("html [%s] [%s] +[%s]",
        moment.duration(moment(snippet.savingTime) - moment() ).humanize(true),
        _.keys(
            _.omit(snippet, [
                '_id', 'savingTime', 'id', 'userId',
{'fraction_non_alphanumeric': 0.17006088599622088, 'fraction_numerical': 0.0027293722443837917, 'mean_word_length': 2.71606864274571, 'pattern_counts': {'":': 8, '<': 5, '<?xml version=': 0, '>': 3, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6397023', 'n_tokens_mistral': 1440, 'n_tokens_neox': 1392, 'n_words': 327}