<commit_before>var fs = require( 'fs' ),
    Twit = require( 'twit' ),
    _ = require( 'underscore' )._;

var actors = [],
    minWords = 50000,
    script = [],
    stats = {
        requests: 0,
        tweets: 0,
        words: 0
    },
    twitter;

var titles = [
    "All's Well That Tweets Well",
    "As You Tweet It",
    "The Comedy of Twitter",
    "Love's Labour's Tweet",
    "Tweet for Tweet",
    "The Merchant of Twitter",
    "The Merry Wives of Twitter",
    "A Midsummer Night's Tweet",
    "Much Ado About Twitter",
    "The Tweeting of the Shrew",
    "The Tweet",
    "Twelfth Tweet",
    "The Two Gentlemen of Twitter",
    "The Two Noble Tweeters",
    "The Winter's Tweet",
    "%USER% and %USER%",
    "%USER% of Twitter",
    "The Tragedy of %USER%, Prince of Twitter",
    "King %USER%",
    "%USER% IV, Part I",
    "%USER% IV, Part II"
];

fs.readFile( 'credentials.json', 'utf8', function( error, data ) {
    if( error ) {
        console.error( error + '\nSee README.md for instructions.' );
        return;
    }

    twitter = new Twit( JSON.parse( data ) );

    twitter.get(
        'search/tweets',
        { q: 'alas -filter:links', lang: 'en', count: 10 },
        function( error, reply ) {
            stats.requests++;
            if( error ) {
                throw new Error( error );
            }

            var tweet, chosenData;

            // choose the first tweet that fits the bill
            _.each( reply.statuses, function( data ) {
                if( !tweet ) {
                    tweet = validateAndClean( data );
                    chosenData = data;
                }
            });

            if( !tweet ) {
                console.error( "Suitable seed tweet wasn't found, please try again in a while." );
                return;
            }

            addActor( tweet.actor );
            setActorInfo( chosenData.user );

            // Pull 200 tweets from the user and validate them
            getTweets( tweet.actor, main )
        }
    );
});


/**
 * The main loop that collects the text
 * 
 * @param newTweets
 */
function main( newTweets ) {
    _.each( newTweets, function( tweet ) {
        script.push( tweet );
        stats.words += countWords( tweet.text );
    });
}


/**
 * Add an actor to the actor list and make sure it's unique
 *
 * @param actor string
 */
function addActor( actor ) {
    // if the actor is already seen, do nothing
    if( _.contains( actors, actor ) || _.where( actors, { name: actor } ).length ) {
        return;
    }

    actors.push( actor );
}


function countWords( text ) {
    return text.split( ' ' ).length;
}


/**
 * Get a user's latest tweets.
 *
 * @param actor
 * @param callback function A callback function called after the request is done.
 *  The AJAX response object is given as
 */
function getTweets( actor, callback ) {
    twitter.get(
        'statuses/user_timeline',
        { screen_name: actor, count: 200 },
        function( error, reply ) {<commit_msg>Day 5<commit_after>var fs = require( 'fs' ),
    Twit = require( 'twit' ),
    _ = require( 'underscore' )._;

var actors = [],
    minWords = 50000,
    script = [],
    stats = {
        requests: 0,
        tweets: 0,
        words: 0
    },
    twitter;

var titles = [
    "All's Well That Tweets Well",
    "As You Tweet It",
    "The Comedy of Twitter",
    "Love's Labour's Tweet",
    "Tweet for Tweet",
    "The Merchant of Twitter",
    "The Merry Wives of Twitter",
    "A Midsummer Night's Tweet",
    "Much Ado About Twitter",
    "The Tweeting of the Shrew",
    "The Tweet",
    "Twelfth Tweet",
    "The Two Gentlemen of Twitter",
    "The Two Noble Tweeters",
    "The Winter's Tweet",
    "%USER% and %USER%",
    "%USER% of Twitter",
    "The Tragedy of %USER%, Prince of Twitter",
    "King %USER%",
    "%USER% IV, Part I",
    "%USER% IV, Part II"
];

fs.readFile( 'credentials.json', 'utf8', function( error, data ) {
    if( error ) {
        console.error( error + '\nSee README.md for instructions.' );
        return;
    }

    twitter = new Twit( JSON.parse( data ) );

    twitter.get(
        'search/tweets',
        { q: 'alas -filter:links', lang: 'en', count: 10 },
        function( error, reply ) {
            stats.requests++;
            if( error ) {
                throw new Error( error );
            }

            var tweet, chosenData;

            // choose the first tweet that fits the bill
            _.each( reply.statuses, function( data ) {
                if( !tweet ) {
                    tweet = validateAndClean( data );
                    chosenData = data;
                }
            });

            if( !tweet ) {
                console.error( "Suitable seed tweet wasn't found, please try again in a while." );
                return;
            }

            addActor( tweet.actor );
            setActorInfo( chosenData.user );

            // Pull 200 tweets from the user and validate them
            getTweets( tweet.actor, main )
        }
    );
});


/**
 * The main loop that collects the text
 * 
 * @param newTweets
 */
function main( newTweets ) {
    _.each( newTweets, function( tweet ) {
        script.push( tweet );
        stats.words += countWords( tweet.text );
    });

    if( stats.words >= minWords ) {

    }

    getNextActor( function( nextActor ) {
        if( nextActor === undefined ) {
            throw new Error( "Didn't find enough actors to generate " + minWords + " words, stopping at " + stats.words + " words." );
        }

        nextActor.appeared = true;

        getTweets( nextActor, main );
    });
}


/**
 * Add an actor to the actor list and make sure it's unique
 *
 * @param actor string
 */
function addActor( actor ) {
    // if the actor is already seen, do nothing
    if( _.contains( actors, actor ) || _.where( actors, { name: actor } ).length ) {
        return;
    }

    actors.push( actor );
}


/**
 * Count words of a string
 *
 * @param text
 * @returns {Number}
 */
function countWords( text ) {
    return text.split( ' ' ).length;
}


/**
 * Fetch more specific user data from Twitter
 */
function fetchActorData() {

}


function getNextActor( callback ) {
    var index = _.find( actors, function( data ) {
        return typeof data === 'string' || data.appeared === false;
    });

    // if the actor is a string, process the actors to turn it into an object
    if( typeof actors[ index ] === 'string' ) {
        fetchActorData();
        getNextActor( callback );
        return;
    }

    callback( actors[ index ].screen_name );
}


/**
 * Get a user's latest tweets.
 *
 * @param actor
 * @param callback function A callback function called after the request is done.
 *  The AJAX response object is given as
 */
function getTweets( actor, callback ) {
    twitter.get(
        'statuses/user_timeline',
        { screen_name: actor, count: 200 },
        function( error, reply ) {
{'fraction_non_alphanumeric': 0.1265841013824885, 'fraction_numerical': 0.005040322580645161, 'mean_word_length': 1.9168416631667367, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 4, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5726263', 'n_tokens_mistral': 2204, 'n_tokens_neox': 2063, 'n_words': 691}