<commit_before>var config = require('config')
  , twitter = require('twitter')
  , _ = require('underscore')._
  ;

var util = require('util');

var Follower = require('../model/follower');
var User = require('../model/user');

var twit = new twitter(config.twitter);

var handleError = function (err) {
  if (err) return console.error(err);
};

var onPreviewDatas = function (err, screen_name, follower, doc, callback) {
  if (err) {
    console.error(err);
    follower.save(handleError);
    return callback(follower);
  }
  if (!doc) {
    console.log('No history for: ' + screen_name);
    follower.save(handleError);
    return callback(follower);
  }
  follower.new_followers = _.difference(follower.followers, doc.followers);
  follower.lost_followers = _.difference(doc.followers, follower.followers);
  follower.save(handleError);
  return callback(follower);
};

var onFollowersReceived = function (screen_name, data, callback) {
  var follower = new Follower({
    screen_name: screen_name,
    created_at: new Date(),
    followers: data['ids']
  });
  Follower.findOne({
    screen_name: screen_name
  }, {
    followers: 1
  }).lean().sort({
      created_at: -1
    }).exec(function (err, doc) {
      onPreviewDatas(err, screen_name, follower, doc, callback);
    });
};

var updateUserScreenName = function(ids) {
  User.find({
    "_id": { "$in": ids }
  }, {
    "_id": 1
  }, function(err, docs) {
    if(err) return console.error(err);
    var known = _.reduce(docs, function(memo, doc) { return memo.concat(doc._id); }, []);
    var unknown = _.difference(ids, known);
    if(unknown.length == 0) return;
    if(unknown.length > 100) {
      unknown = unknown.splice(0,100);
    }
    unknown = unknown.join(',');
    if(unknown.length > 1024) {
      unknown = unknown.substr(0,1024);
      unknown = unknown.substr(0,unknown.lastIndexOf(','));
    }
    twit.get('/users/lookup.json', {
      user_id: unknown,
      inlude_entities: false
    }, function(data) {
      if(typeof data != 'object') return handleError(data);
      data.forEach(function(row) {
        User.update({
          _id: row.id
        }, {
          screen_name: row.screen_name,
          name: row.name,
          description: row.description,
          created_at: new Date()
        }, { upsert: true }, handleError);
      });
    });
  });
};

module.exports.updateUserScreenName = updateUserScreenName;

module.exports.refresh = function (req, res) {
  if (!req.body.screen_name) return res.send(400, { "message": "You must provide a screen_name." });
  module.exports.doRefresh(req.body.screen_name, function(err, follower) {
    res.send(200, {
      screen_name: follower.screen_name,
      followers_count: (follower.followers ? follower.followers.length : 0),
      lost_followers: follower.lost_followers,
      new_followers: follower.new_followers
    });
  });

};

module.exports.doRefresh = function (screen_name, callback) {
  var onStats = function (follower) {
    callback(null, follower);
    var ids = follower.lost_followers.concat(follower.new_followers);
    updateUserScreenName(ids);
  };
  twit.get('/followers/ids.json', {
    screen_name: screen_name,
    cursor: -1
  }, function (data) {
    onFollowersReceived(screen_name, data, onStats);
  });
};
<commit_msg>Fix error detection from Twitter API<commit_after>var config = require('config')
  , twitter = require('twitter')
  , _ = require('underscore')._
  ;

var util = require('util');

var Follower = require('../model/follower');
var User = require('../model/user');

var twit = new twitter(config.twitter);

var handleError = function (err) {
  if (err) return console.error(err);
};

var onPreviewDatas = function (err, screen_name, follower, doc, callback) {
  if (err) {
    console.error(err);
    follower.save(handleError);
    return callback(follower);
  }
  if (!doc) {
    console.log('No history for: ' + screen_name);
    follower.save(handleError);
    return callback(follower);
  }
  follower.new_followers = _.difference(follower.followers, doc.followers);
  follower.lost_followers = _.difference(doc.followers, follower.followers);
  follower.save(handleError);
  return callback(follower);
};

var onFollowersReceived = function (screen_name, data, callback) {
  var follower = new Follower({
    screen_name: screen_name,
    created_at: new Date(),
    followers: data['ids']
  });
  Follower.findOne({
    screen_name: screen_name
  }, {
    followers: 1
  }).lean().sort({
      created_at: -1
    }).exec(function (err, doc) {
      onPreviewDatas(err, screen_name, follower, doc, callback);
    });
};

var updateUserScreenName = function(ids) {
  User.find({
    "_id": { "$in": ids }
  }, {
    "_id": 1
  }, function(err, docs) {
    if(err) return console.error(err);
    var known = _.reduce(docs, function(memo, doc) { return memo.concat(doc._id); }, []);
    var unknown = _.difference(ids, known);
    if(unknown.length == 0) return;
    if(unknown.length > 100) {
      unknown = unknown.splice(0,100);
    }
    unknown = unknown.join(',');
    if(unknown.length > 1024) {
      unknown = unknown.substr(0,1024);
      unknown = unknown.substr(0,unknown.lastIndexOf(','));
    }
    twit.get('/users/lookup.json', {
      user_id: unknown,
      inlude_entities: false
    }, function(data) {
      if(!Array.isArray(data)) return handleError(data);
      data.forEach(function(row) {
        User.update({
          _id: row.id
        }, {
          screen_name: row.screen_name,
          name: row.name,
          description: row.description,
          created_at: new Date()
        }, { upsert: true }, handleError);
      });
    });
  });
};

module.exports.updateUserScreenName = updateUserScreenName;

module.exports.refresh = function (req, res) {
  if (!req.body.screen_name) return res.send(400, { "message": "You must provide a screen_name." });
  module.exports.doRefresh(req.body.screen_name, function(err, follower) {
    res.send(200, {
      screen_name: follower.screen_name,
      followers_count: (follower.followers ? follower.followers.length : 0),
      lost_followers: follower.lost_followers,
      new_followers: follower.new_followers
    });
  });

};

module.exports.doRefresh = function (screen_name, callback) {
  var onStats = function (follower) {
    callback(null, follower);
    var ids = follower.lost_followers.concat(follower.new_followers);
    updateUserScreenName(ids);
  };
  twit.get('/followers/ids.json', {
    screen_name: screen_name,
    cursor: -1
  }, function (data) {
    onFollowersReceived(screen_name, data, onStats);
  });
};

{'fraction_non_alphanumeric': 0.15058965830057455, 'fraction_numerical': 0.008769277290595706, 'mean_word_length': 3.357707509881423, 'pattern_counts': {'":': 8, '<': 3, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3901819', 'n_tokens_mistral': 2258, 'n_tokens_neox': 2132, 'n_words': 429}