<commit_before>/**
 * Everything to do with fetching tweets from Twitter, doing things to the
 * tweets, and sending them to the front end.
 *
 * Get it going with:
 *  var streamer = require('./app/streamer')(settings, twitter, sockets);
 *  streamer.start();
 */
module.exports = function(settings, twitter, sockets, _) {

  var streamer = this;

  /**
   * For each category, its most recent tweets, newest last.
   * eg
   * {
   *   'uk': [{...}, {...}, {...}],
   *   'us': ...
   * }
   */
  streamer.cache = {};


  /**
   * Set everything going.
   * First get existing tweets, then start listening for new ones.
   */
  streamer.start = function() {
    streamer.twitter = new twitter({
      consumer_key: settings.twitter.consumer_key,
      consumer_secret: settings.twitter.consumer_secret,
      access_token_key: settings.twitter.access_token_key,
      access_token_secret: settings.twitter.access_token_secret 
    });

    // These methods are kind of dependent on each other, so we only run them
    // serially.
    streamer.queue = [
                      streamer.get_user_ids,
                      streamer.cache_previous_tweets,
                      streamer.start_streaming];

    // Let's start...
    streamer.next_in_queue();
  };


  // Call this to call the next method in the queue.
  streamer.next_in_queue = function() {
    if (streamer.queue.length > 0) {
      (streamer.queue.shift())();
    };
  };

      
  /**
   * Get all the Twitter user IDs from the screen_names we have in config.
   * We need the user IDs, not screen_names, for streaming.
   */
  streamer.get_user_ids = function() {
    console.log('Fetching Twitter user IDs');
    streamer.twitter.lookupUser(settings.watched_screen_names, function(err, users) {
      if (err) {
        console.log("Error fetching user IDs: "+err);
      } else {
        users.forEach(function(user){
          settings.watched_ids.push(user.id); 
        });
        // On to the next method...
        streamer.next_in_queue();
      }; 
    });
  };


  /**
   * Get some recent tweets for each user account, so that we have some tweets
   * to display from the start.
   */
  streamer.cache_previous_tweets = function() {
    console.log('Caching previous tweets');

    settings.watched_ids.forEach(function(id) {
      // We don't apply a count, because that doesn't include retweets and
      // @replies.
      streamer.twitter.getUserTimeline({
        user_id: id, 
        trim_user: false, exclude_replies: true,
        contributor_details: true, include_rts: false
      }, function(err, tweets) {
        if (err) {
          console.log("Error fetching tweets for user id '"+id+"': "+err); 
        } else {
          tweets.forEach(function(tweet){
            streamer.add_tweet_to_cache(tweet);
          });
          // On to the next method...
          streamer.next_in_queue();
        };
      })
    });
  };


  /**<commit_msg>Improve the number of tweets we fetch initially.<commit_after>/**
 * Everything to do with fetching tweets from Twitter, doing things to the
 * tweets, and sending them to the front end.
 *
 * Get it going with:
 *  var streamer = require('./app/streamer')(settings, twitter, sockets);
 *  streamer.start();
 */
module.exports = function(settings, twitter, sockets, _) {

  var streamer = this;

  /**
   * For each category, its most recent tweets, newest last.
   * eg
   * {
   *   'uk': [{...}, {...}, {...}],
   *   'us': ...
   * }
   */
  streamer.cache = {};


  /**
   * Set everything going.
   * First get existing tweets, then start listening for new ones.
   */
  streamer.start = function() {
    streamer.twitter = new twitter({
      consumer_key: settings.twitter.consumer_key,
      consumer_secret: settings.twitter.consumer_secret,
      access_token_key: settings.twitter.access_token_key,
      access_token_secret: settings.twitter.access_token_secret 
    });

    // These methods are kind of dependent on each other, so we only run them
    // serially.
    streamer.queue = [
                      streamer.get_user_ids,
                      streamer.cache_previous_tweets,
                      streamer.start_streaming];

    // Let's start...
    streamer.next_in_queue();
  };


  // Call this to call the next method in the queue.
  streamer.next_in_queue = function() {
    if (streamer.queue.length > 0) {
      (streamer.queue.shift())();
    };
  };

      
  /**
   * Get all the Twitter user IDs from the screen_names we have in config.
   * We need the user IDs, not screen_names, for streaming.
   */
  streamer.get_user_ids = function() {
    console.log('Fetching Twitter user IDs');
    streamer.twitter.lookupUser(settings.watched_screen_names, function(err, users) {
      if (err) {
        console.log("Error fetching user IDs: "+err);
      } else {
        users.forEach(function(user){
          settings.watched_ids.push(user.id); 
        });
        // On to the next method...
        streamer.next_in_queue();
      }; 
    });
  };


  /**
   * Get some recent tweets for each user account, so that we have some tweets
   * to display from the start.
   */
  streamer.cache_previous_tweets = function() {
    console.log('Caching previous tweets');

    settings.watched_ids.forEach(function(id) {
      // Note: The 'count' doesn't include retweets and replies, so we'll
      // probably get less than that many tweets returned.
      streamer.twitter.getUserTimeline({
        user_id: id, 
        trim_user: false, exclude_replies: true,
        contributor_details: true, include_rts: false, count: 200
      }, function(err, tweets) {
        if (err) {
          console.log("Error fetching tweets for user id '"+id+"': "+err); 
        } else {
          tweets.forEach(function(tweet){
            streamer.add_tweet_to_cache(tweet);
          });
          // On to the next method...
          streamer.next_in_queue();
        };
      })
    });
  };


  /**
{'fraction_non_alphanumeric': 0.12374413931681179, 'fraction_numerical': 0.0008372404554588078, 'mean_word_length': 2.276467361492046, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3897320', 'n_tokens_mistral': 1855, 'n_tokens_neox': 1764, 'n_words': 477}