<commit_before>// Fetcher, gets the HTML page and returns the wanted data. 

'use strict';

var async = require('async');
var moment = require('moment');
var levelup = require('level');
var path = require('path');
var request = require('request');
var winston = require('winston');

var parser = require('./parser');

module.exports = function (startFrom) {
  // Database to keep track of already fetched content.
  // The keys are the date, and the value is the data structure returned
  // by the parser. LevelUP transparently translates it to and from JSON.
  var db = levelup(path.resolve(__dirname, './mordac.db'), { 
    valueEncoding: 'json',
  });

  // Save the point we started from.
  var startpoint = startFrom;
  
  // Keep track of where we are now.
  var position = startpoint;

  // Get data for a specific date.
  var fetchDate = function (date, callback) {
    // First, try getting it from the database.
    db.get(date, function (err, value) {
      // If we do not have data for this date yet, fetch the markup,
      // parse it and store the output.
      if ((err && err.name === 'NotFoundError') ||
          !value.largeImage || !value.largeImage.url) {
        async.waterfall([
          function (cb) {
            fetchMarkup(date, cb);
          },

          // The parser takes the date and markup returned by fetchMarkup.
          parser,

          // Save the extracted data before proceeding.
          function (date, values, cb) {
            db.put(date, values, function (err) {
              cb(err, date, values);
            });
          }
        ], callback);
      }
      else {
        callback(err, date, value);
      }
    });
  };

  // Fetch markup for a specific date.
  var fetchMarkup = function (date, callback) {
    var URL = 'http://dilbert.com/' + date;

    winston.info('Fetching ' + URL);

    request(URL, function (error, response, body) {
      return callback(error, date, body);
    });
  };

  this.run = function (dayCount, callback) {
    var stopDate = moment(startpoint).subtract('days', 30);

    var dateValues = {};

    // Iterate sequentially over each of the days, unless an error occurs.
    async.whilst(
      function () { return position.isAfter(stopDate); },
      function (cb) {
        fetchDate(position.format('YYYY-MM-DD'), function (err, date, values) {
          dateValues[date] = values;

          cb(err, date, values);
        });

        // Subtract a day so we move on to the previous one.
        position.subtract(1, 'days');
      },
      function (err) {
        if (err) {
          winston.error(err);
        }

        callback(err, dateValues);
      }
    );
  };

  return this;
};
<commit_msg>More error resilience.<commit_after>// Fetcher, gets the HTML page and returns the wanted data. 

'use strict';

var async = require('async');
var moment = require('moment');
var levelup = require('level');
var path = require('path');
var request = require('request');
var winston = require('winston');

var parser = require('./parser');

module.exports = function (startFrom) {
  // Database to keep track of already fetched content.
  // The keys are the date, and the value is the data structure returned
  // by the parser. LevelUP transparently translates it to and from JSON.
  var db = levelup(path.resolve(__dirname, './mordac.db'), { 
    valueEncoding: 'json',
  });

  // Save the point we started from.
  var startpoint = startFrom;
  
  // Keep track of where we are now.
  var position = startpoint;

  // Get data for a specific date.
  var fetchDate = function (date, callback) {
    // First, try getting it from the database.
    db.get(date, function (err, value) {
      // If we do not have data for this date yet, fetch the markup,
      // parse it and store the output.
      if ((err && err.name === 'NotFoundError') || !value ||
          !value.largeImage || !value.largeImage.url) {
        async.waterfall([
          function (cb) {
            fetchMarkup(date, cb);
          },

          // The parser takes the date and markup returned by fetchMarkup.
          parser,

          // Save the extracted data before proceeding.
          function (date, values, cb) {
            if (values) {
              db.put(date, values, function (err) {
                cb(err, date, values);
              });
            }
            else {
              cb();
            }
          }
        ], callback);
      }
      else {
        callback(err, date, value);
      }
    });
  };

  // Fetch markup for a specific date.
  var fetchMarkup = function (date, callback) {
    var URL = 'http://dilbert.com/' + date;

    winston.info('Fetching ' + URL);

    request(URL, function (error, response, body) {
      return callback(error, date, body);
    });
  };

  this.run = function (dayCount, callback) {
    var stopDate = moment(startpoint).subtract('days', dayCount);

    var dateValues = {};

    // Iterate sequentially over each of the days, unless an error occurs.
    async.whilst(
      function () { return position.isAfter(stopDate); },
      function (cb) {
        fetchDate(position.format('YYYY-MM-DD'), function (err, date, values) {
          if (values) {
            dateValues[date] = values;
          }

          cb(err, date, values);
        });

        // Subtract a day so we move on to the previous one.
        position.subtract(1, 'days');
      },
      function (err) {
        if (err) {
          winston.error(err);
        }

        callback(err, dateValues);
      }
    );
  };

  return this;
};

{'fraction_non_alphanumeric': 0.12661174785100288, 'fraction_numerical': 0.0007163323782234957, 'mean_word_length': 2.1376404494382024, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5118915', 'n_tokens_mistral': 1621, 'n_tokens_neox': 1569, 'n_words': 537}