<commit_before>/**
 * WorkerController
 *
 * @description :: Server-side logic for managing Workercontrollers
 * @help        :: See http://sailsjs.org/#!/documentation/concepts/Controllers
 */


var Promise = require('bluebird');
var _ = require('lodash');

module.exports = {

  processMessage: function(req, res) {
    var type = req.headers['x-aws-sqsd-attr-type'];
    var body =  req.body;

    if (type === 'topic') {
      var packageName = req.body.package.package;
      var packageVersion = req.body.package.version;
      var result = Topic.createWithRdFile({input: req.body, packageName: packageName, packageVersion: packageVersion});
      result.then(function(value) {
        key = 'view_topic_' + value.id;
        RedisClient.del(key);
        res.json(value);
      })
      .catch(Sequelize.UniqueConstraintError, function (err) {
        return res.send(409, err.errors);
      }).catch(Sequelize.ValidationError, function (err) {
        return res.send(400, err.errors);
      }).catch(function(err){
          return res.negotiate(err.errors);
      });

    } else if (type === 'version') {
      return sails.controllers.packageversion.postDescription(req, res);
    } else {
      res.send(400, 'Invalid type');
    }
  },

  indexStats: function(req, res) {
    CronService.indexAggregatedDownloadStats().then(function(result) {
      console.log("Finished indexing stats");
      res.send(200, "done");
    }).catch(function(err){
      return res.negotiate(err.errors);
    });
  },

  lastDaySplittedDownloads: function(req, res) {
    DownloadStatistic.findLastIndexedDay().then(function(lastDay) {
      var lastDate = new Date(lastDay);
      var now = new Date();
      var diff = Utils.dateDiffInDays(lastDate, now);
      return diff;
    }).then(function(nDays) {
      if (nDays <= 1) {
        console.log("Nothing new");
        return res.send(200, "done");
      }
      var range = _.range(1, nDays);
      Promise.map(range, function (nDay) {
        return CronService.splittedAggregatedDownloadstats(nDay)
          .catch({message: "empty"}, function() {
            console.log("No stats for this time range yet");
            return 1;
          });
      }, {concurrency: 1})
      .then(function (result) {
        console.log("Finished indexing splitted stats");
        res.send(200, "done");
      }).catch(function(err) {
        return res.negotiate(err);
      });
<commit_msg>fix worker when no stats for last day yet<commit_after>/**
 * WorkerController
 *
 * @description :: Server-side logic for managing Workercontrollers
 * @help        :: See http://sailsjs.org/#!/documentation/concepts/Controllers
 */


var Promise = require('bluebird');
var _ = require('lodash');

module.exports = {

  processMessage: function(req, res) {
    var type = req.headers['x-aws-sqsd-attr-type'];
    var body =  req.body;

    if (type === 'topic') {
      var packageName = req.body.package.package;
      var packageVersion = req.body.package.version;
      var result = Topic.createWithRdFile({input: req.body, packageName: packageName, packageVersion: packageVersion});
      result.then(function(value) {
        key = 'view_topic_' + value.id;
        RedisClient.del(key);
        res.json(value);
      })
      .catch(Sequelize.UniqueConstraintError, function (err) {
        return res.send(409, err.errors);
      }).catch(Sequelize.ValidationError, function (err) {
        return res.send(400, err.errors);
      }).catch(function(err){
          return res.negotiate(err.errors);
      });

    } else if (type === 'version') {
      return sails.controllers.packageversion.postDescription(req, res);
    } else {
      res.send(400, 'Invalid type');
    }
  },

  indexStats: function(req, res) {
    CronService.indexAggregatedDownloadStats().then(function(result) {
      console.log("Finished indexing stats");
      res.send(200, "done");
    }).catch(function(err){
      return res.negotiate(err.errors);
    });
  },

  lastDaySplittedDownloads: function(req, res) {
    DownloadStatistic.findLastIndexedDay().then(function(lastDay) {
      var lastDate = new Date(lastDay);
      var now = new Date();
      var diff = Utils.dateDiffInDays(lastDate, now);
      return diff;
    }).then(function(nDays) {
      if (nDays <= 1) {
        console.log("Nothing new");
        return res.send(200, "done");
      }
      var range = _.range(1, nDays);
      Promise.map(range, function (nDay) {
        console.log("Started indexing for today - " + nDay + "days");
        return CronService.splittedAggregatedDownloadstats(nDay)
          .catch({message: "empty"}, function() {
            console.log("No stats for this time range yet");
            return 1;
          })
          .catch(function(err) {
            console.log("Undefined response");
            return 1;
          });
      }, {concurrency: 1})
      .then(function (result) {
        console.log("Finished indexing splitted stats");
        res.send(200, "done");
      }).catch(function(err) {
        return res.negotiate(err);
      });

{'fraction_non_alphanumeric': 0.14538658272673619, 'fraction_numerical': 0.008853039543576628, 'mean_word_length': 2.716374269005848, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 8, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2118799', 'n_tokens_mistral': 1589, 'n_tokens_neox': 1518, 'n_words': 315}