<commit_before>#!/usr/bin/env node
'use strict';

/**
 * Update only expired packages (those with nextUpdate previous to current date)
 * (C) 2015 <NAME>.
 */

// requires
require('prototypes');
var config = require('../config.js');
var packages = require('../lib/packages.js');
var estimator = require('../lib/estimation.js');
var async = require('async');
var moment = require('moment');
var Log = require('log');
var testing = require('testing');

// globals
var log = new Log(config.logLevel);
var limit = config.limit;

/**
 * Go over all the packages in all.json and update mongo if required.
 * There are two constraints:
 *   - Github only allows 5000 requests per hour.
 *   - We should use config.limit to avoid RangeErrors in the stack size
 * @param all[required]: the object including the packages to update, preserving the all.json format.
 * @param callback[required]: a function(error, result) with the results of the process
 */
exports.goOver = function(all, callback)
{
	var names = Object.keys(all);
	var numberOfPackages = names.length;
	limit = (limit === null) || (numberOfPackages < limit) ? numberOfPackages : limit;
	var chunks = [];
	for (var j = 0; j < Math.ceil(numberOfPackages/limit); j++)
	{
		chunks.push([]);
	}
	var packageCount = 0;
	for (var name in all)
	{
		var entry = all[name];
		var index = Math.floor(packageCount/limit);
		chunks[index].push(getEstimator(entry));
		packageCount++;
		delete all[name];
	}
	log.debug('number of chunks: ' + chunks.length);
	var series = [];
	while (chunks.length)
	{
		series.push(getChunkProcessor(chunks.shift()));
	}
	log.debug('series has length ' + series.length);
	async.series(series, function(error)
	{
		if (error)
		{
			callback(error);
		}
		callback(null, series.length);
	});
};

function getEstimator(entry)
{
    return function(callback)
    {
        var name = entry.name;
        var now = moment();
        packages.find(name, function (error, item)
        {
            var isNewEntry = error || !item;
            var shouldUpdate = isNewEntry || (moment(item.nextUpdate) < now);
            // return if we should not update
            if (!shouldUpdate)
            {
                log.info('Discarding package: ' + name);
                return callback(null);
            }
            // update!!
            estimator.estimate(entry, function (error, estimation)
            {
                // new entry?
                if (!isNewEntry && estimation)
                {
                    delete estimation.created;
                    estimation.timesUpdated = item.timesUpdated + 1;
                    // should we defer the update one year
                    var created = moment(item.created);
                    var lastUpdated = moment(estimation.lastUpdated);
                    var monthsAgo = lastUpdated.diff(created, 'months');
                    if ((monthsAgo > 11) && (estimation.timesUpdated <= (monthsAgo + 1)))
                    {
                        estimation.nextUpdate = moment(lastUpdated).add(1, 'years');
                    }
                }
                return callback (error, estimation);
            });
        });
    };<commit_msg>Return early on error.<commit_after>#!/usr/bin/env node
'use strict';

/**
 * Update only expired packages (those with nextUpdate previous to current date)
 * (C) 2015 <NAME>.
 */

// requires
require('prototypes');
var config = require('../config.js');
var packages = require('../lib/packages.js');
var estimator = require('../lib/estimation.js');
var async = require('async');
var moment = require('moment');
var Log = require('log');
var testing = require('testing');

// globals
var log = new Log(config.logLevel);
var limit = config.limit;

/**
 * Go over all the packages in all.json and update mongo if required.
 * There are two constraints:
 *   - Github only allows 5000 requests per hour.
 *   - We should use config.limit to avoid RangeErrors in the stack size
 * @param all[required]: the object including the packages to update, preserving the all.json format.
 * @param callback[required]: a function(error, result) with the results of the process
 */
exports.goOver = function(all, callback)
{
	var names = Object.keys(all);
	var numberOfPackages = names.length;
	limit = (limit === null) || (numberOfPackages < limit) ? numberOfPackages : limit;
	var chunks = [];
	for (var j = 0; j < Math.ceil(numberOfPackages/limit); j++)
	{
		chunks.push([]);
	}
	var packageCount = 0;
	for (var name in all)
	{
		var entry = all[name];
		var index = Math.floor(packageCount/limit);
		chunks[index].push(getEstimator(entry));
		packageCount++;
		delete all[name];
	}
	log.debug('number of chunks: ' + chunks.length);
	var series = [];
	while (chunks.length)
	{
		series.push(getChunkProcessor(chunks.shift()));
	}
	log.debug('series has length ' + series.length);
	async.series(series, function(error)
	{
		if (error)
		{
			callback(error);
		}
		callback(null, series.length);
	});
};

function getEstimator(entry)
{
    return function(callback)
    {
        var name = entry.name;
        var now = moment();
        packages.find(name, function (error, item)
        {
			if (error)
			{
				return callback(error);
			}
            var shouldUpdate = !item || (moment(item.nextUpdate) < now);
            // return if we should not update
            if (!shouldUpdate)
            {
                log.info('Discarding package: ' + name);
                return callback(null);
            }
            // update!!
            estimator.estimate(entry, function (error, estimation)
            {
				if (error)
				{
					return callback(error);
				}
                // new entry?
                if (item && estimation)
                {
                    delete estimation.created;
                    estimation.timesUpdated = item.timesUpdated + 1;
                    // should we defer the update one year
                    var created = moment(item.created);
                    var lastUpdated = moment(estimation.lastUpdated);
                    var monthsAgo = lastUpdated.diff(created, 'months');
                    if ((monthsAgo > 11) && (estimation.timesUpdated <= (monthsAgo + 1)))
                    {
                        estimation.nextUpdate = moment(lastUpdated).add(1, 'years');
                    }
                }
                return callback (null, estimation);
            });
        });
    };
{'fraction_non_alphanumeric': 0.1291323917429769, 'fraction_numerical': 0.004656216048424647, 'mean_word_length': 2.256189994946943, 'pattern_counts': {'":': 0, '<': 13, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4460162', 'n_tokens_mistral': 1911, 'n_tokens_neox': 1778, 'n_words': 511}