<commit_before>var opbeat = require('opbeat')({ active: process.env.NODE_ENV === 'production' });
var crypto = require('crypto');
var request = require('request');
var mongojs = require('mongojs');
var cheerio = require('cheerio');
var sendgrid = require('sendgrid')(process.env.SENDGRID_USERNAME, process.env.SENDGRID_PASSWORD);

var db = mongojs(process.env.MONGO_URI || 'localhost/website-watcher', ['config', 'log']);
db.log.ensureIndex({ url: 1, hash: 1 });

var notify = function (config, url, a, b) {
  config.notify.forEach(function (email) {
    sendgrid.send({
      to: email,
      from: config.from,
      subject: 'Update to watched page',
      text: 'The following URL have changed:\n' + url,
      files: [
        { filename: 'old.html', contentType: 'text/html', content: a },
        { filename: 'new.html', contentType: 'text/html', content: b }
      ]
    }, function (err, json) {
      if (err) return opbeat.captureError(err);
      if (json.message !== 'success') return opbeat.captureError(new Error('Unexpected SendGrid response'), { extra: json });
      console.log('Sent notification to', email);
    });
  });
};

var loadConfig = function (cb) {
  console.log('Loading config...');
  db.config.find(function (err, docs) {
    if (err) return cb(err);
    if (!docs.length) return cb(new Error('No config found! Please create a single MongoDB document in the config collection'));
    cb(null, docs[0]);
  });
};

var fingerprint = function (data) {
  var shasum = crypto.createHash('sha1');
  shasum.update(data);
  return shasum.digest('hex');
};

var checkUrl = function (url, query, cb) {
  console.log('Fetching:', url);
  request(url, function (err, res, html) {
    if (err) return cb(err);

    $ = cheerio.load(html);
    var content = $(query).html();
    if (typeof content !== 'string') return cb(new Error('Could not extract content from html'));
    content = new Buffer(content);
    var hash = fingerprint(content);

    db.log.find({ url: url }).sort({ when: -1 }).limit(1, function (err, docs) {
      if (err) return cb(err);
      var old = docs.length ? docs[0] : { html: { buffer: new Buffer('') }, content: { buffer: new Buffer('') } };
      if (hash === fingerprint(old.content.buffer)) return cb();
      console.log('Detected change:', url);
      db.log.insert({ url: url, hash: hash, when: new Date(), html: new Buffer(html), content: content }, function (err) {
        cb(err, old.html.buffer.toString(), html);
      });
    });
  });
};

var run = function (cb) {
  loadConfig(function (err, config) {
    if (err) {
      opbeat.captureError(err);
      cb();
      return;
    }

    var callbacks = config.sites.length;
    config.sites.forEach(function (site) {
      checkUrl(site.url, site.query, function (err, a, b) {
        if (err) opbeat.captureError(err, { extra: { url: site.url, query: site.query } });
        if (a || b) notify(config, site.url, a, b);
        if (!--callbacks) cb();
      });
    });
  });
};

var agent = function () {
  run(function () {
    console.log('Hibernating for 10 minutes...');
    setTimeout(run, 1000*60*10);
  });
};

agent();<commit_msg>Ensure the agent keeps running<commit_after>var opbeat = require('opbeat')({ active: process.env.NODE_ENV === 'production' });
var crypto = require('crypto');
var request = require('request');
var mongojs = require('mongojs');
var cheerio = require('cheerio');
var sendgrid = require('sendgrid')(process.env.SENDGRID_USERNAME, process.env.SENDGRID_PASSWORD);

var db = mongojs(process.env.MONGO_URI || 'localhost/website-watcher', ['config', 'log']);
db.log.ensureIndex({ url: 1, hash: 1 });

var notify = function (config, url, a, b) {
  config.notify.forEach(function (email) {
    sendgrid.send({
      to: email,
      from: config.from,
      subject: 'Update to watched page',
      text: 'The following URL have changed:\n' + url,
      files: [
        { filename: 'old.html', contentType: 'text/html', content: a },
        { filename: 'new.html', contentType: 'text/html', content: b }
      ]
    }, function (err, json) {
      if (err) return opbeat.captureError(err);
      if (json.message !== 'success') return opbeat.captureError(new Error('Unexpected SendGrid response'), { extra: json });
      console.log('Sent notification to', email);
    });
  });
};

var loadConfig = function (cb) {
  console.log('Loading config...');
  db.config.find(function (err, docs) {
    if (err) return cb(err);
    if (!docs.length) return cb(new Error('No config found! Please create a single MongoDB document in the config collection'));
    cb(null, docs[0]);
  });
};

var fingerprint = function (data) {
  var shasum = crypto.createHash('sha1');
  shasum.update(data);
  return shasum.digest('hex');
};

var checkUrl = function (url, query, cb) {
  console.log('Fetching:', url);
  request(url, function (err, res, html) {
    if (err) return cb(err);

    $ = cheerio.load(html);
    var content = $(query).html();
    if (typeof content !== 'string') return cb(new Error('Could not extract content from html'));
    content = new Buffer(content);
    var hash = fingerprint(content);

    db.log.find({ url: url }).sort({ when: -1 }).limit(1, function (err, docs) {
      if (err) return cb(err);
      var old = docs.length ? docs[0] : { html: { buffer: new Buffer('') }, content: { buffer: new Buffer('') } };
      if (hash === fingerprint(old.content.buffer)) return cb();
      console.log('Detected change:', url);
      db.log.insert({ url: url, hash: hash, when: new Date(), html: new Buffer(html), content: content }, function (err) {
        cb(err, old.html.buffer.toString(), html);
      });
    });
  });
};

var run = function (cb) {
  loadConfig(function (err, config) {
    if (err) {
      opbeat.captureError(err);
      cb();
      return;
    }

    var callbacks = config.sites.length;
    config.sites.forEach(function (site) {
      checkUrl(site.url, site.query, function (err, a, b) {
        if (err) opbeat.captureError(err, { extra: { url: site.url, query: site.query } });
        if (a || b) notify(config, site.url, a, b);
        if (!--callbacks) cb();
      });
    });
  });
};

var agent = function () {
  run(function () {
    console.log('Hibernating for 10 minutes...');
    setTimeout(agent, 1000*60*10);
  });
};

agent();
{'fraction_non_alphanumeric': 0.17822720808974563, 'fraction_numerical': 0.00537209669774056, 'mean_word_length': 3.3385880740233036, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4058496', 'n_tokens_mistral': 2070, 'n_tokens_neox': 2020, 'n_words': 559}