<commit_before>var schedule = require('node-schedule');
var Promise = require('bluebird');
var pgp = require('pg-promise')({ promiseLib: Promise });
var _ = require('lodash');
var logger = require('log4js').getLogger('scheduled');
var exec = require('child-process-promise').exec;
var Queue = require('promise-queue');

var crawler = require('./crawler/crawler.js');
var params = require('./crawler/src/params.js');
var generator = require('./tile-generator/generator.js');

Queue.configure(Promise);

var db = pgp(params.dbConnStr);

db.any('SELECT id, name, url, georeferenced, crawl_schedule FROM instance')
  .then(function(instances) {

    _.forEach(instances, function(instance) {
      schedule.scheduleJob(instance.crawl_schedule, function(){
        var queue = new Queue(1, Infinity, {
          onEmpty: function() {
            return crawler.refresh(db)
              .then(function() { return generator.preseed(instance.id); })
              .then(function() { return exec('pm2 restart odd.tile-server'); })
              .then(function() { process.exit(); });
          }
        });

        crawler.crawl(instance.name, instance.id, instance.url, instance.georeferenced, queue)
        .catch(function(err) {
          logger.error(err);
        });

      });
    });
  });<commit_msg>Update scheduled.js<commit_after>var schedule = require('node-schedule');
var Promise = require('bluebird');
var pgp = require('pg-promise')({ promiseLib: Promise });
var _ = require('lodash');
var logger = require('log4js').getLogger('scheduled');
var exec = require('child-process-promise').exec;
var Queue = require('promise-queue');

var crawler = require('./crawler/crawler.js');
var params = require('./crawler/src/params.js');
var generator = require('./tile-generator/generator.js');

Queue.configure(Promise);

var db = pgp(params.dbConnStr);

db.each('SELECT id, name, url, georeferenced, crawl_schedule FROM instance', [], function (instance) {
    schedule.scheduleJob(instance.crawl_schedule, function () {
        var queue = new Queue(1, Infinity, {
            onEmpty: function () {
                return crawler.refresh(db)
                    .then(function () {
                        return generator.preseed(instance.id);
                    })
                    .then(function () {
                        return exec('pm2 restart odd.tile-server');
                    })
                    .then(function () {
                        process.exit();
                    });
            }
        });
        crawler.crawl(instance.name, instance.id, instance.url, instance.georeferenced, queue)
            .catch(function (err) {
                logger.error(err);
            });

    });
})
    .catch(function (err) {
        logger.error(err);
    });
{'fraction_non_alphanumeric': 0.15555555555555556, 'fraction_numerical': 0.002150537634408602, 'mean_word_length': 2.5373891001267426, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2072375', 'n_tokens_mistral': 801, 'n_tokens_neox': 777, 'n_words': 152}