<commit_before>'use strict';

var crypto = require('crypto');
var request = require('request');
var cheerio = require('cheerio');
var db = require('./db');

var fingerprint = function (data) {
  var shasum = crypto.createHash('sha1');
  shasum.update(data || '');
  return shasum.digest('hex');
};

var getExcerpt = function (data, query) {
  if (!data) return null;
  var $ = cheerio.load(data);
  var excerpt = $(query).html();
  if (typeof excerpt !== 'string') return null;
  return new Buffer(excerpt);
};

var getNewData = function (url, cb) {
  console.log('Fetching:', url);
  request(url, function (err, res, data) {
    if (err) return cb(err);
    if (res.statusCode >= 300) return cb(new Error('Unexpected HTTP status code: ' + res.statusCode));
    cb(null, new Buffer(data));
  });
};

var getOldData = function (url, cb) {
  db.log.find({ url: url }).sort({ when: -1 }).limit(1, function (err, docs) {
    if (err) return cb(err);
    if (!docs.length) return cb();
    cb(null, docs[0].data.buffer, docs[0].when);
  });
};

module.exports = function (url, query, cb) {
  getNewData(url, function (err, data) {
    if (err) return cb(err);
    getOldData(url, function (err, oldData, when) {
      if (err) return cb(err);
      if (query) {
        var excerpt = getExcerpt(data, query);
        var oldExcerpt = getExcerpt(oldData, query);
        if (!excerpt) return cb(new Error('Could not get excerpt from new data'));
        if (oldData && !oldExcerpt) return cb(new Error('Could not get excerpt from old data'));
      }

      var hash = fingerprint(excerpt || data);<commit_msg>Add debugging data to error<commit_after>'use strict';

var crypto = require('crypto');
var request = require('request');
var cheerio = require('cheerio');
var db = require('./db');

var fingerprint = function (data) {
  var shasum = crypto.createHash('sha1');
  shasum.update(data || '');
  return shasum.digest('hex');
};

var getExcerpt = function (data, query) {
  if (!data) return null;
  var $ = cheerio.load(data);
  var excerpt = $(query).html();
  if (typeof excerpt !== 'string') return null;
  return new Buffer(excerpt);
};

var getNewData = function (url, cb) {
  console.log('Fetching:', url);
  request(url, function (err, res, data) {
    if (err) return cb(err);
    if (res.statusCode >= 300) return cb(new Error('Unexpected HTTP status code: ' + res.statusCode));
    cb(null, new Buffer(data));
  });
};

var getOldData = function (url, cb) {
  db.log.find({ url: url }).sort({ when: -1 }).limit(1, function (err, docs) {
    if (err) return cb(err);
    if (!docs.length) return cb();
    cb(null, docs[0].data.buffer, docs[0].when);
  });
};

module.exports = function (url, query, cb) {
  getNewData(url, function (err, data) {
    if (err) return cb(err);
    getOldData(url, function (err, oldData, when) {
      if (err) return cb(err);
      if (query) {
        var excerpt = getExcerpt(data, query);
        var oldExcerpt = getExcerpt(oldData, query);
        if (!excerpt) {
          err = new Error('Could not get excerpt from new data');
          err.bodySize = data.length;
          cb(err);
          return;
        }
        if (oldData && !oldExcerpt) return cb(new Error('Could not get excerpt from old data'));
      }

      var hash = fingerprint(excerpt || data);
{'fraction_non_alphanumeric': 0.16747133373566686, 'fraction_numerical': 0.004828002414001207, 'mean_word_length': 3.414114513981358, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3161129', 'n_tokens_mistral': 1132, 'n_tokens_neox': 1076, 'n_words': 329}