<commit_before>var basicScraper = require('./controllers/basicScraperController');
var getExternalUrl = require('./controllers/urlController').getExternalUrl;
var CronJob = require('cron').CronJob;
var secret = require('../config.js');
var db = require('./db.js');
var Sequelize = require('sequelize');
var mandrill = require('mandrill-api');
mandrill_client = new mandrill.Mandrill(secret.mandrill.client_id);
var compare = require('./imgCompare.js').compare;

// To run the cronjob as it is now: navigate to server dir and type node cronjob
// for every five minutes
// var schedule = '* +' */5 * * * *';

// for faster testing
var schedule = '*/5 * * * * *';
//To run every 3 seconds do */3; every 5 min do * */5 *

var cronjob = new CronJob(schedule, function() {
  console.log('You will see this message every 5 min');
  // check database for jobs assigned for cronjob

  // get urls
  db.User.findAll()
  .then(function(allUsers) {
    for (var i = 0; i < allUsers.length; i++){
      var currEmail = allUsers[i].email;
      console.log('email', currEmail)
      allUsers[i].getUrls()
      .then(function(url) {
        console.log('in the for loop')
        for (var j=0; j<url.length; j++){
         console.log('url', url[j].UserUrl.cropImage)
           // console.log('url', url[j].id)

           var img1 = url[j].UserUrl.cropImage;
           var params = {
            h: url[j].UserUrl.cropHeight,
            w: url[j].UserUrl.cropWidth,
            x: url[j].UserUrl.cropOriginX,
            y: url[j].UserUrl.cropOriginY
          }
           //console.log('about to call basicScraper')
        // get the server to render the page with params coordinates
        basicScraper.getScreenshot(url[j].url, url[j].id, function(urlToThePage) {
          console.log('url to the page', urlToThePage)
          basicScraper.cropImg(urlToThePage, params, true,  function(img2) {
            console.log('image path', img2);
            compare(img1, img2);
            // sendEmail(currEmail, currEmail);
          });
        });
        }
      });
    };
  });
}, null, true, 'America/Los_Angeles');

var sendEmail = function (email, name){
  var message = {
    "html": "<span>The Scrapinit found a change in the webpage you are following</span>",
    "subject": "We scraped some tubular stuff for you!!",
    "from_email": email,
    "from_name": "The Blank Arrays",
    "to": [{
      "email": email,
      "name":  name,
      "type": "to"
    },
    ],
    "headers": {
      "Reply-To": ""
    },
    "important": true,
  };
<commit_msg>[Feature] set up cronjob to compare images<commit_after>var basicScraper = require('./controllers/basicScraperController');
var getExternalUrl = require('./controllers/urlController').getExternalUrl;
var CronJob = require('cron').CronJob;
var secret = require('../config.js');
var db = require('./db.js');
var Sequelize = require('sequelize');
var mandrill = require('mandrill-api');
mandrill_client = new mandrill.Mandrill(secret.mandrill.client_id);
var compare = require('./imgCompare.js').compare;

// To run the cronjob as it is now: navigate to server dir and type node cronjob
// for every five minutes
// var schedule = '* +' */5 * * * *';

// for faster testing
var schedule = '*/5 * * * * *';
//To run every 3 seconds do */3; every 5 min do * */5 *

var cronjob = new CronJob(schedule, function() {
  console.log('You will see this message every 5 min');
  // check database for jobs assigned for cronjob

  // get urls
  db.User.findAll()
  .then(function(allUsers) {
    for (var i = 0; i < allUsers.length; i++){
      var currEmail = allUsers[i].email;
      console.log('email', currEmail)
      allUsers[i].getUrls()
      .then(function(url) {
        for (var j=0; j<url.length; j++){
          console.log('checking', url[j].url, 'for changes');

           var img1 = url[j].UserUrl.cropImage;
           var params = {
            h: url[j].UserUrl.cropHeight,
            w: url[j].UserUrl.cropWidth,
            x: url[j].UserUrl.cropOriginX,
            y: url[j].UserUrl.cropOriginY
          }
           //console.log('about to call basicScraper')
        // get the server to render the page with params coordinates
        basicScraper.getScreenshot(url[j].url, url[j].id, function(urlToThePage) {
          basicScraper.cropImg(urlToThePage, params, true, function(img2) {
            console.log('old image path', img1);
            console.log('new image path', img2);
          });
        });
        }
      });
    };
  });
}, null, true, 'America/Los_Angeles');

var sendEmail = function (email, name){
  var message = {
    "html": "<span>The Scrapinit found a change in the webpage you are following</span>",
    "subject": "We scraped some tubular stuff for you!!",
    "from_email": email,
    "from_name": "The Blank Arrays",
    "to": [{
      "email": email,
      "name":  name,
      "type": "to"
    },
    ],
    "headers": {
      "Reply-To": ""
    },
    "important": true,
  };

{'fraction_non_alphanumeric': 0.15511022044088177, 'fraction_numerical': 0.005410821643286573, 'mean_word_length': 2.643065693430657, 'pattern_counts': {'":': 22, '<': 11, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4693725', 'n_tokens_mistral': 1626, 'n_tokens_neox': 1581, 'n_words': 422}