<commit_before>'use strict';

var googleapis = require('googleapis');
var async = require('async');

var config = require('../config/configuration.js');
var retrieveThreads = require('./helpers/retrieve.js');

module.exports = function updateAccount(serviceData, cursor, queues, cb) {
  // Retrieve all mails since last call
  async.waterfall([
    function refreshTokens(cb) {
      var oauth2Client = new googleapis.auth.OAuth2(config.googleId, config.googleSecret, serviceData.callbackUrl);
      oauth2Client.refreshToken_(serviceData.tokens.refresh_token, function(err, tokens) {
        if(err) {
          return cb(err);
        }

        if(typeof tokens !== 'object' || !tokens.access_token) {
          return cb(new Error("Can't refresh tokens"));
        }

        tokens.refresh_token = serviceData.tokens.refresh_token;
        serviceData.tokens = tokens;
        cb(err);
      });
    },
    function getThreads(cb) {
      var options = {
        maxResults: 1000,
        userId: serviceData.accountName
      };

      if(!cursor) {
        cursor = {
          date: new Date(1970),
          id: 0
        };
      }

      var oauth2Client = new googleapis.auth.OAuth2(config.googleId, config.googleSecret, serviceData.callbackUrl);
      oauth2Client.credentials = serviceData.tokens;
      options.auth = oauth2Client;

      retrieveThreads(options, cursor, [], {}, cb);
    },
    function addThreadsToQueue(newCursor, threads, cb) {<commit_msg>Send TokenError when we need a refresh<commit_after>'use strict';

var googleapis = require('googleapis');
var async = require('async');
var TokenError = require('anyfetch-provider').TokenError;

var config = require('../config/configuration.js');
var retrieveThreads = require('./helpers/retrieve.js');

module.exports = function updateAccount(serviceData, cursor, queues, cb) {
  // Retrieve all mails since last call
  async.waterfall([
    function refreshTokens(cb) {
      var oauth2Client = new googleapis.auth.OAuth2(config.googleId, config.googleSecret, serviceData.callbackUrl);
      oauth2Client.refreshToken_(serviceData.tokens.refresh_token, function(err, tokens) {
        if(err) {
          if(err.toString().match(/token/i)) {
            return cb(new TokenError());
          }
          return cb(err);
        }

        if(typeof tokens !== 'object' || !tokens.access_token) {
          return cb(new TokenError());
        }

        tokens.refresh_token = serviceData.tokens.refresh_token;
        serviceData.tokens = tokens;
        cb(err);
      });
    },
    function getThreads(cb) {
      var options = {
        maxResults: 1000,
        userId: serviceData.accountName
      };

      if(!cursor) {
        cursor = {
          date: new Date(1970),
          id: 0
        };
      }

      var oauth2Client = new googleapis.auth.OAuth2(config.googleId, config.googleSecret, serviceData.callbackUrl);
      oauth2Client.credentials = serviceData.tokens;
      options.auth = oauth2Client;

      retrieveThreads(options, cursor, [], {}, cb);
    },
    function addThreadsToQueue(newCursor, threads, cb) {
{'fraction_non_alphanumeric': 0.12259306803594351, 'fraction_numerical': 0.010269576379974325, 'mean_word_length': 2.8058608058608057, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3163597', 'n_tokens_mistral': 909, 'n_tokens_neox': 887, 'n_words': 185}