<commit_before>/**
 * Contains the entry point for the node.js app. This...
 * - loads the user's OAuth data
 * - opens a hanging request to the user's stream
 * - pieces together the image processing / Tweet re-uploading pipeline
 *
 * All application logic should be in this file. The pipeline is pieced together
 * using an 'args' object that is passed around per-Tweet, since ain't nobody
 * got time to line up arguments to callbacks.
 * 
 * The basic pipeline is.. 
 * runUserStream | chunkReaderCallback | downloadImage | processImage | 
 *     uploadImage | sendTweetReplyWithMedia
 */

var appUtil = require('./apputil.js');
var config = require('./config.js');
var https = require('https');
// $FlowFixMe: Type check lwip.
var lwip = require('lwip');
var responses = require('./net/responses.js');
var twitter = require('./twitter/twitter.js');
var twitterJson = require('./twitter/json.js');
var util = require('util');


/**
 * Opens the user stream, passing all chunks into the constructed chunk
 * aggregator. This aggregator kicks off the processing pipeline for each tweet.
 */
var runUserStream = function(): void {
  console.log('opening user stream for user ' + config.username);
  var chunkReader = new twitter.DelimitedChunkAggregator(chunkReaderCallback);

  var request = twitter.openUserStream(config.oauthData, function(response) {
    console.log('user stream status code: ' + response.statusCode);
    
    response.on('data', function(data) {
      chunkReader.add(data);
    });
    response.on('end', function() {
      console.error('Connection was ended by Twitter.');
    });

    if (response.statusCode != 200) {
      console.error('Failed to connect to Twitter.');
    }
  });

  request.on('error', function(e) {
    console.error('Error connecting to user stream: ', e.message, util.inspect(e));
  });
  request.end();
};


/**
 * Attempts to parse Tweet data from the given buffer, and then downloads the
 * first image URL it finds in the Tweet that is a supported format, and process
 * that image.
 */
var chunkReaderCallback = function(err: ?Error, buffer: ?Buffer) {
  if (err) {
    console.error('Received bad chunk data. Crashing.');
    throw err;
  }
  if (!buffer) {
    throw Error('Expected non-null buffer for chunk reader callback.');
  }
  console.log('Received complete chunk.');
  var tweet = JSON.parse(buffer.toString('utf8'));

  if (!twitterJson.isTweet(tweet)) {
    console.log('Received non-tweet message.');
    return;
  }

  var screenName = twitterJson.getTweetScreenName(tweet);
  if (!screenName) {
    console.error('Unrecognizable Tweet format', util.inspect(tweet));
    return;
  }
  if (screenName.toLowerCase() == config.username.toLowerCase()) {
    // Avoid processing a steady stream of Tweets from self.
    return;
  }

  var urls = twitterJson.getTweetPhotoUrls(tweet).filter(
    appUtil.urlHasSupportedImageType);
  if (urls.length) {
    var args = {
      oauthKeyData: config.oauthData,
      imageUrl: urls[0],
      tweet: tweet
    };
    downloadImage(args);
  } else {
    console.log('No photo URLs in tweet.');
  }
};


/**
 * Downloads the given image, and attempts to open it in LWIP. If successful,
 * passes the image onto processImage.
 */
var downloadImage = function(args: Object): void {
  console.log('Downloading image.');
  var url = args.imageUrl;
  console.log('Downloading image ' + url);

  // $FlowIssue: https module is not defined.
  var request = https.get(url, function(response) {
    console.log('Download image status code: ' + response.statusCode);
    if (response.statusCode != 200) {
      console.error('Error downloading image.');
      responses.collectBuffer(response, function(data) {
        console.error('Error response: ', data.toString('utf8'));
      });
      return;
    }
    responses.collectBuffer(response, function(data) {
      lwip.open(data, url.substr(url.lastIndexOf('.') + 1).toLowerCase(), 
        function(err, image) {
          if (err) {
            console.error('opening the image did not work. ' + err);
            return;
          }
          console.log('Image downloaded and opened successfully.');
          args.image = image;
          processImage(args);
        });
    });
  });
  request.on('error', function(e) {
    console.error('Error downloading image', e);
  });
  request.end();
};<commit_msg>Ony requests small versions of images from Twitter.<commit_after>/**
 * Contains the entry point for the node.js app. This...
 * - loads the user's OAuth data
 * - opens a hanging request to the user's stream
 * - pieces together the image processing / Tweet re-uploading pipeline
 *
 * All application logic should be in this file. The pipeline is pieced together
 * using an 'args' object that is passed around per-Tweet, since ain't nobody
 * got time to line up arguments to callbacks.
 * 
 * The basic pipeline is.. 
 * runUserStream | chunkReaderCallback | downloadImage | processImage | 
 *     uploadImage | sendTweetReplyWithMedia
 */

var appUtil = require('./apputil.js');
var config = require('./config.js');
var https = require('https');
// $FlowFixMe: Type check lwip.
var lwip = require('lwip');
var responses = require('./net/responses.js');
var twitter = require('./twitter/twitter.js');
var twitterJson = require('./twitter/json.js');
var util = require('util');


/**
 * Opens the user stream, passing all chunks into the constructed chunk
 * aggregator. This aggregator kicks off the processing pipeline for each tweet.
 */
var runUserStream = function(): void {
  console.log('opening user stream for user ' + config.username);
  var chunkReader = new twitter.DelimitedChunkAggregator(chunkReaderCallback);

  var request = twitter.openUserStream(config.oauthData, function(response) {
    console.log('user stream status code: ' + response.statusCode);
    
    response.on('data', function(data) {
      chunkReader.add(data);
    });
    response.on('end', function() {
      console.error('Connection was ended by Twitter.');
    });

    if (response.statusCode != 200) {
      console.error('Failed to connect to Twitter.');
    }
  });

  request.on('error', function(e) {
    console.error('Error connecting to user stream: ', e.message, util.inspect(e));
  });
  request.end();
};


/**
 * Attempts to parse Tweet data from the given buffer, and then downloads the
 * first image URL it finds in the Tweet that is a supported format, and process
 * that image.
 */
var chunkReaderCallback = function(err: ?Error, buffer: ?Buffer) {
  if (err) {
    console.error('Received bad chunk data. Crashing.');
    throw err;
  }
  if (!buffer) {
    throw Error('Expected non-null buffer for chunk reader callback.');
  }
  console.log('Received complete chunk.');
  var tweet = JSON.parse(buffer.toString('utf8'));

  if (!twitterJson.isTweet(tweet)) {
    console.log('Received non-tweet message.');
    return;
  }

  var screenName = twitterJson.getTweetScreenName(tweet);
  if (!screenName) {
    console.error('Unrecognizable Tweet format', util.inspect(tweet));
    return;
  }
  if (screenName.toLowerCase() == config.username.toLowerCase()) {
    // Avoid processing a steady stream of Tweets from self.
    return;
  }

  var urls = twitterJson.getTweetPhotoUrls(tweet).filter(
    appUtil.urlHasSupportedImageType);
  if (urls.length) {
    var args = {
      oauthKeyData: config.oauthData,
      imageUrl: urls[0],
      tweet: tweet
    };
    downloadImage(args);
  } else {
    console.log('No photo URLs in tweet.');
  }
};


/**
 * Downloads the given image, and attempts to open it in LWIP. If successful,
 * passes the image onto processImage.
 */
var downloadImage = function(args: Object): void {
  console.log('Downloading image.');
  var url = args.imageUrl;
  var smallUrl = url + ':small';
  console.log('Downloading image ' + smallUrl);

  // $FlowIssue: https module is not defined.
  var request = https.get(smallUrl, function(response) {
    console.log('Download image status code: ' + response.statusCode);
    if (response.statusCode != 200) {
      console.error('Error downloading image.');
      responses.collectBuffer(response, function(data) {
        console.error('Error response: ', data.toString('utf8'));
      });
      return;
    }
    responses.collectBuffer(response, function(data) {
      lwip.open(data, url.substr(url.lastIndexOf('.') + 1).toLowerCase(), 
        function(err, image) {
          if (err) {
            console.error('opening the image did not work. ' + err);
            return;
          }
          console.log('Image downloaded and opened successfully.');
          args.image = image;
          processImage(args);
        });
    });
  });
  request.on('error', function(e) {
    console.error('Error downloading image', e);
  });
  request.end();
};
{'fraction_non_alphanumeric': 0.13170731707317074, 'fraction_numerical': 0.0022688598979013048, 'mean_word_length': 3.306790425012213, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 22, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3663221', 'n_tokens_mistral': 2585, 'n_tokens_neox': 2533, 'n_words': 763}