<commit_before>#!/usr/bin/env node
require('dotenv').config();

const _ = require('lodash');
const fs = require('fs-extra');
const cli = require('cli');
const Twitter = require('twitter');

const username = 'everycolorbot';

/**
 * Creates a new instance of the Twitter client.
 *
 * @type {Twitter}
 */
const client = new Twitter({
    consumer_key: process.env.TWITTER_API_KEY,
    consumer_secret: process.env.TWITTER_API_SECRET,
    access_token_key: process.env.TWITTER_ACCESS_TOKEN,
    access_token_secret: process.env.TWITTER_ACCESS_TOKEN_SECRET,
});

/**
 * Synchronously handles an asynchronous promise.
 *
 * @param {function} promise
 */
const synchronousPromiseHandler = promise => {
    let iterator = promise();
    let loop = result => {
        !result.done && result.value.then(
            res => loop(iterator.next(res)),
            err => loop(iterator.throw(err))
        );
    };

    loop(iterator.next());
};

/**
 * Parses a collection of Tweets into a friendly array.
 *
 * @param {Array} tweets
 * @returns {Array}
 */
const parseTweetCollection = (tweets) => {
    let collection = [];

    for (const tweet of tweets) {
        collection.push({
            id: tweet.id_str,
            color: tweet.text.split(' ')[0].replace('0x', '#'),
            interactions: tweet.favorite_count + tweet.retweet_count,
        });
    }

    return collection;
};

/**
 * Parses through all available Tweets and stores the results in a JSON file.
 */
synchronousPromiseHandler(function* () {
    let params = {
        count: 200,
        trim_user: true,
        include_rts: false,
        exclude_replies: true,
        screen_name: username,
    };

    let collection = [];

    try {
        for (let i = 0; i < 250; i++) {
            collection = collection.concat(parseTweetCollection(yield client.get('statuses/user_timeline', params)));
            params.max_id = collection[collection.length - 1].id;

            cli.progress(i / 100)
        }

        cli.progress(1);

        // Use outputFile instead of outputJson so that we get a minified file.
        fs.outputFile(
            `${__dirname}/dist/colors.json`,
            JSON.stringify(_.sortBy(collection, 'interactions').reverse())
        );
    } catch (err) {
        console.log(err.message);
    }<commit_msg>Add some debug messages<commit_after>#!/usr/bin/env node
require('dotenv').config();

const _ = require('lodash');
const fs = require('fs-extra');
const cli = require('cli');
const Twitter = require('twitter');

const username = 'everycolorbot';

/**
 * Creates a new instance of the Twitter client.
 *
 * @type {Twitter}
 */
const client = new Twitter({
    consumer_key: process.env.TWITTER_API_KEY,
    consumer_secret: process.env.TWITTER_API_SECRET,
    access_token_key: process.env.TWITTER_ACCESS_TOKEN,
    access_token_secret: process.env.TWITTER_ACCESS_TOKEN_SECRET,
});

/**
 * Synchronously handles an asynchronous promise.
 *
 * @param {function} promise
 */
const synchronousPromiseHandler = promise => {
    let iterator = promise();
    let loop = result => {
        !result.done && result.value.then(
            res => loop(iterator.next(res)),
            err => loop(iterator.throw(err))
        );
    };

    loop(iterator.next());
};

/**
 * Parses a collection of Tweets into a friendly array.
 *
 * @param {Array} tweets
 * @returns {Array}
 */
const parseTweetCollection = (tweets) => {
    let collection = [];

    for (const tweet of tweets) {
        let color = tweet.text.split(' ')[0].replace('0x', '#');
        let interactions = tweet.favorite_count + tweet.retweet_count;

        cli.debug(`Color: ${color}, Interactions: ${interactions}`);

        collection.push({
            id: tweet.id_str,
            color: color,
            interactions: interactions,
        });
    }

    return collection;
};

/**
 * Parses through all available Tweets and stores the results in a JSON file.
 */
synchronousPromiseHandler(function* () {
    let params = {
        count: 200,
        trim_user: true,
        include_rts: false,
        exclude_replies: true,
        screen_name: username,
    };

    let collection = [];

    try {
        cli.info(`Trawling @${username}'s Tweets`);

        for (let i = 0; i < 100; i++) {
            collection = collection.concat(parseTweetCollection(yield client.get('statuses/user_timeline', params)));
            params.max_id = collection[collection.length - 1].id;

            cli.progress(i / 100)
        }

        cli.progress(1)
    } catch (err) {
        cli.error(err[0].message);
    } finally {
        cli.info(`Trawling complete - ${collection.length} Tweets found`);
        cli.info(`Saving Tweets to: ${__dirname}/dist/colors.json`);

        // Use outputFile instead of outputJson so that we get a minified file.
        fs.outputFileSync(
            `${__dirname}/dist/colors.json`,
            JSON.stringify(_.sortBy(collection, 'interactions').reverse())
        );
    }
{'fraction_non_alphanumeric': 0.13636363636363635, 'fraction_numerical': 0.0058069683620344415, 'mean_word_length': 2.614327062228654, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 13, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '270893', 'n_tokens_mistral': 1555, 'n_tokens_neox': 1481, 'n_words': 352}