<commit_before>var Twit = require('twit');
var pubsub = require('@google-cloud/pubsub')();
var storage = require('@google-cloud/storage')();
var cmudict;

exports.fetch = function(req, res) {
  var topic = pubsub.topic('tweets');
  var bucket = storage.bucket('longest-poem-in-the-world.appspot.com');
  bucket.file('twitter.json').download(function(err, contents) {
    if (err) {
      return res.status(500).send(err);
    }

    var credentials = JSON.parse(contents);
    fetchTweets(credentials, function(tweets, err) {
      if (err) {
        return res.status(500).send(err);
      }

      console.log("Fetched " + tweets.length + " tweets.");

      bucket.file('cmudict.json').download(function(err, contents) {
        cmudict = JSON.parse(contents);
        var candidates = tweets.filter(hasPotential);

        console.log("Filtered to " + candidates.length + " candidates.");

        if (candidates.length > 0) {
          topic.publish(candidates, function(err) {
            if (err) {
              return res.status(500).send(err);
            }

            res.status(200).send('OK with ' + candidates.length + ' candidates');
          });
        } else {
          res.status(200).send('OK but no candidates');
        }
      });
    });
  });
}

function fetchTweets(credentials, callback) {
  var searchTerms = ["and", "the", "am", "i", "is", "to"];

  var T = new Twit({
    consumer_key : credentials.consumer_key,
    consumer_secret : credentials.consumer_secret,
    access_token : credentials.access_token,
    access_token_secret : credentials.access_token_secret
  });

  T.get('search/tweets', {
    q: searchTerms.join(" OR "),
    count: 100,
    langauge: 'en'
  }, function(err, data, response) {
    if (err) {
      return callback(null, err);<commit_msg>Make sure we only fetch recent tweets, not popular ones<commit_after>var Twit = require('twit');
var pubsub = require('@google-cloud/pubsub')();
var storage = require('@google-cloud/storage')();
var cmudict;

exports.fetch = function(req, res) {
  var topic = pubsub.topic('tweets');
  var bucket = storage.bucket('longest-poem-in-the-world.appspot.com');
  bucket.file('twitter.json').download(function(err, contents) {
    if (err) {
      return res.status(500).send(err);
    }

    var credentials = JSON.parse(contents);
    fetchTweets(credentials, function(tweets, err) {
      if (err) {
        return res.status(500).send(err);
      }

      console.log("Fetched " + tweets.length + " tweets.");

      bucket.file('cmudict.json').download(function(err, contents) {
        cmudict = JSON.parse(contents);
        var candidates = tweets.filter(hasPotential);

        console.log("Filtered to " + candidates.length + " candidates.");

        if (candidates.length > 0) {
          topic.publish(candidates, function(err) {
            if (err) {
              return res.status(500).send(err);
            }

            res.status(200).send('OK with ' + candidates.length + ' candidates');
          });
        } else {
          res.status(200).send('OK but no candidates');
        }
      });
    });
  });
}

function fetchTweets(credentials, callback) {
  var searchTerms = ["and", "the", "am", "i", "is", "to"];

  var T = new Twit({
    consumer_key : credentials.consumer_key,
    consumer_secret : credentials.consumer_secret,
    access_token : credentials.access_token,
    access_token_secret : credentials.access_token_secret
  });

  T.get('search/tweets', {
    q: searchTerms.join(" OR "),
    count: 100,
    result_type: "recent",
    langauge: 'en'
  }, function(err, data, response) {
    if (err) {
      return callback(null, err);
{'fraction_non_alphanumeric': 0.15671845189424913, 'fraction_numerical': 0.010357045516489506, 'mean_word_length': 2.863157894736842, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '233169', 'n_tokens_mistral': 1202, 'n_tokens_neox': 1139, 'n_words': 224}