<commit_before>'use strict';

const tokenDealer = require('token-dealer');
const log = require('npmlog');
const values = require('lodash/values');
const minBy = require('lodash/minBy');

/**
 * Monitors the tokens managed by token-dealer of a given group.
 *
 * @param {array}  tokens  The array of tokens
 * @param {string} [group] The token's group (e.g.: github)
 */
function statTokens(tokens, group) {
    // Do nothing if loglevel is higher than stat
    if (log.levels[log.level] < log.level.stat) {
        return;
    }

    setInterval(() => {
        const tokensUsage = values(tokenDealer.getTokensUsage(tokens, { group }));
        const usableTokensUsage = tokensUsage.filter((entry) => !entry.exhausted);

        if (usableTokensUsage.length) {
            log.stat('tokens', `${usableTokensUsage.length} out of ${tokensUsage.length} tokens are usable (${group})`);
            return;
        }

        const nextResettingToken = minBy(tokensUsage, 'reset');
        const remainingMins = Math.ceil((nextResettingToken.reset - Date.now()) / 1000 / 60);<commit_msg>Don't fail if we have no tokens<commit_after>'use strict';

const tokenDealer = require('token-dealer');
const log = require('npmlog');
const values = require('lodash/values');
const minBy = require('lodash/minBy');

/**
 * Monitors the tokens managed by token-dealer of a given group.
 *
 * @param {array}  tokens  The array of tokens
 * @param {string} [group] The token's group (e.g.: github)
 */
function statTokens(tokens, group) {
    // Do nothing if loglevel is higher than stat
    if (log.levels[log.level] < log.level.stat) {
        return;
    }

    setInterval(() => {
        const tokensUsage = values(tokenDealer.getTokensUsage(tokens, { group }));
        const usableTokensUsage = tokensUsage.filter((entry) => !entry.exhausted);

        if (usableTokensUsage.length) {
            log.stat('tokens', `${usableTokensUsage.length} out of ${tokensUsage.length} tokens are usable (${group})`);
            return;
        }
        if (tokensUsage.length < 1) {
            log.stat('tokens', `We have no tokens (${group})`);
            return;
        }

        const nextResettingToken = minBy(tokensUsage, 'reset');
        const remainingMins = Math.ceil((nextResettingToken.reset - Date.now()) / 1000 / 60);
{'fraction_non_alphanumeric': 0.14088159031979255, 'fraction_numerical': 0.0056179775280898875, 'mean_word_length': 3.1339285714285716, 'pattern_counts': {'":': 0, '<': 6, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7023930', 'n_tokens_mistral': 703, 'n_tokens_neox': 681, 'n_words': 185}