<commit_before>#!/usr/bin/env node
'use strict';

/**
 * Go over all packages and estimate quality for all.
 * (C) 2014 <NAME>.
 */

// requires
require('prototypes');
var config = require('../config.js');
var db = require('../lib/db.js');
var estimator = require('../lib/estimation.js');
var async = require('async');
var moment = require('moment');
var Log = require('log');

// globals
var log = new Log(config.logLevel);
//var githubApiRemainingCalls;
//var githubApiResetLimit;
var limit = config.limit;
var packagesCollection;

/**
 * Go over all the packages in all.json and update mongo.
 * There are two constraints:
 *   - Github only allows 5000 requests per hour.
 *   - We should use config.limit to avoid RangeErrors in the stack size
 * @param offset[optional]: offset over the all.json file. Defaults to zero.
 * @param callback[required]: a function(error, result) with the results of the process
 */
exports.goOver = function(offset, callback)
{
    if (typeof offset === 'function') {
        callback = offset;
        offset = 0;
    }
    var all;
    log.info('loading all.json...');
    try
    {
        all = require('../all.json');
        delete all._updated;
    }
    catch(exception)
    {
        return callback('Could not parse all.json: ' + exception);
    }
    log.info('all.json loaded');
    var names = Object.keys(all);
    log.info('All packages: ' + names.length);
    if (offset)
    {
        log.info('Offset ' + offset);
        for (var i=0; i<offset; i++)
        {
            delete all[names.shift()];
        }
    }
    var numberOfPackages = names.length;
    log.info('All packages after offset: ' + numberOfPackages);
    limit = (limit === null) || (numberOfPackages < limit) ? numberOfPackages : limit;
    var chunks = [];
    for (var j = 0; j < Math.ceil(numberOfPackages/limit); j++)
    {
        chunks.push([]);
    }
    var packageCount = 0;
    for (var name in all)
    {
        var entry = all[name];
        var index = Math.floor(packageCount/limit);
        chunks[index].push(getEstimator(entry));
        packageCount++;
        delete all[name];
    }
    log.debug('number of chunks: ' + chunks.length);
    db.addCallback(function(error, result)
    {
        if (error) {
            return callback(error);
        }
        packagesCollection = result.collection(config.packagesCollection);
        var series = [];
        while (chunks.length)
        {
            series.push(getChunkProcessor(chunks.shift()));
        }
        log.debug('series has length ' + series.length);
        async.series(series, function(error)
        {
            if (error)
            {
                callback(error);
            }
            callback(null, series.length);
        });
    });
};

function getEstimator(entry)
{
    return function(callback)
    {
        log.info('Estimator about to be called: ' + entry.name);
        estimator.estimate(entry, function (error, result) {
            log.info('Estimation performed for ' + entry.name + ' Error: ', error);
            callback(error, result);
        });
    };
}

function getChunkProcessor(chunk)
{
    log.debug('Returning chunk processor for chunk ' + chunk.length);
    return function(callback)
    {
        log.info('About to process chunk: ' + chunk.length);

        setTimeout(function() {
            log.info('Chunk processed.');
            callback(null);
        }, 2000);
        /*async.parallel(chunk, function(error, estimations)
        {
            if (error)
            {
                log.error('Chunk processed with error ', error);
                return callback(error);
            }
            log.info('Chunk processed.');
            // Adjust pending, remaining calls, etc
            var pendings = [];
            var githubApiRemainingCalls = 9999999;<commit_msg>fake all.js for testing<commit_after>#!/usr/bin/env node
'use strict';

/**
 * Go over all packages and estimate quality for all.
 * (C) 2014 <NAME>.
 */

// requires
require('prototypes');
var config = require('../config.js');
var db = require('../lib/db.js');
var estimator = require('../lib/estimation.js');
var async = require('async');
var moment = require('moment');
var Log = require('log');

// globals
var log = new Log(config.logLevel);
//var githubApiRemainingCalls;
//var githubApiResetLimit;
var limit = config.limit;
var packagesCollection;

/**
 * Go over all the packages in all.json and update mongo.
 * There are two constraints:
 *   - Github only allows 5000 requests per hour.
 *   - We should use config.limit to avoid RangeErrors in the stack size
 * @param offset[optional]: offset over the all.json file. Defaults to zero.
 * @param callback[required]: a function(error, result) with the results of the process
 */
exports.goOver = function(offset, callback)
{
    if (typeof offset === 'function') {
        callback = offset;
        offset = 0;
    }
    var all;
    log.info('loading all.json...');
    try
    {
        all = require('../all.json');
        delete all._updated;
    }
    catch(exception)
    {
        return callback('Could not parse all.json: ' + exception);
    }
    log.info('all.json loaded');
    var names = Object.keys(all);
    log.info('All packages: ' + names.length);
    if (offset)
    {
        log.info('Offset ' + offset);
        for (var i=0; i<offset; i++)
        {
            delete all[names.shift()];
        }
    }
    var numberOfPackages = names.length;
    log.info('All packages after offset: ' + numberOfPackages);
    limit = (limit === null) || (numberOfPackages < limit) ? numberOfPackages : limit;
    var chunks = [];
    for (var j = 0; j < Math.ceil(numberOfPackages/limit); j++)
    {
        chunks.push([]);
    }
    var packageCount = 0;
    for (var name in all)
    {
        var entry = all[name];
        var index = Math.floor(packageCount/limit);
        chunks[index].push(getEstimator(entry));
        packageCount++;
        delete all[name];
    }
    log.debug('number of chunks: ' + chunks.length);
    db.addCallback(function(error, result)
    {
        if (error) {
            return callback(error);
        }
        packagesCollection = result.collection(config.packagesCollection);
        var series = [];
        while (chunks.length)
        {
            series.push(getChunkProcessor(chunks.shift()));
        }
        log.debug('series has length ' + series.length);
        async.series(series, function(error)
        {
            if (error)
            {
                callback(error);
            }
            callback(null, series.length);
        });
    });
};

function getEstimator(entry)
{
    return function(callback)
    {
        log.info('Estimator about to be called: ' + entry.name);
        estimator.estimate(entry, function (error, result) {
            log.info('Estimation performed for ' + entry.name + ' Error: ', error);
            callback(error, result);
        });
    };
}

function getChunkProcessor(chunk)
{
    log.debug('Returning chunk processor for chunk ' + chunk.length);
    return function(callback)
    {
        log.info('About to process chunk: ' + chunk.length);

        setTimeout(function() {
            log.info('Chunk processed.');
            //callback(null);
        }, 2000);

        /*async.parallel(chunk, function(error, estimations)
        {
        });*/

        /*async.parallel(chunk, function(error, estimations)
        {
            if (error)
            {
                log.error('Chunk processed with error ', error);
                return callback(error);
            }
            log.info('Chunk processed.');
            // Adjust pending, remaining calls, etc
            var pendings = [];
            var githubApiRemainingCalls = 9999999;
{'fraction_non_alphanumeric': 0.13541265562828905, 'fraction_numerical': 0.00590424849184957, 'mean_word_length': 2.1843073150796894, 'pattern_counts': {'":': 0, '<': 11, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 2, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1153496', 'n_tokens_mistral': 2254, 'n_tokens_neox': 2192, 'n_words': 577}