<commit_before>/* This module exports the `makeBundle` function that is used to
 * bundle a web page and its resources into data-URIs.
 */

var mime = require('mime');
var async = require('async');
var request = require('superagent');
var cheerio = require('cheerio');
var urllib = require('url');

// Map selectors for cheerio instances to handlers.
var bundleable = {
  'img': replaceImages,
  'link[rel="stylesheet"]': replaceCSSFiles,
  'script': replaceJSFiles
};

function replaceResources(url, html, callback) {
  var $ = cheerio.load(html);
  var selectors = Object.keys(bundleable);
  var functions = [];
  for (var i = 0, len = selectors.length; i < len; ++i) {
    functions.push(function (selector) {
      return function (callback) {
        console.log('Looking for items with selector ' + selector);
        bundleable[selector]($, selector, url, callback);
      };
    }(selectors[i]));
  }
  async.series(functions, function (err, cheerios) {
    if (err) {
      callback(err, null);
    } else {
      console.log('Finished calling series of handlers');
      callback(null, cheerios[cheerios.length - 1].html());
    }
  });
}

function dataURI(url, content) {
  var encoded = content.toString('base64');
  return 'data:' + mime.lookup(url) + ';base64,' + encoded;
}

function fetchAndReplace(attr, elem, url, callback) {
  var resource = elem.attr(attr);
  if (typeof resource === 'undefined' || !resource) {
    return;
  }
  url = urllib.resolve(url, resource);
  request.get(url).end(function (err, result) {
    if (!err) {
      var newuri = dataURI(url, result.body);
      console.log('Computed data uri ' + newuri);
      elem.attr(attr, newuri);
      console.log('Replaced URL');
      callback(null, elem);
    } else {
      callback(err, null);
    }
  });
}

function replaceAll($, selector, url, attr, callback) {
  var elements = [];
  $(selector).each(function (index, elem) {
    var $_this = $(this);
    elements.push($_this);
  });
  async.reduce(elements, elements[0], function (memo, item, next) {
    console.log('Reducing ' + elements.length + ' elements');
    if (typeof memo.attr(attr) === 'undefined') {
      console.log('Skipping element');
      next(null, item);
    } else {
      console.log('Processing new element');
      fetchAndReplace(attr, memo, url, next);
    }
  }, callback);
}

function replaceImages($, selector, url, callback) {
  replaceAll($, selector, url, 'src', callback);
}
<commit_msg>Wrote comments to explain the tricky stuff<commit_after>/* This module exports the `makeBundle` function that is used to
 * bundle a web page and its resources into data-URIs.
 */

var mime = require('mime');
var async = require('async');
var request = require('superagent');
var cheerio = require('cheerio');
var urllib = require('url');

// Map selectors for cheerio instances to handlers.
var bundleable = {
  'img': replaceImages,
  'link[rel="stylesheet"]': replaceCSSFiles,
  'script': replaceJSFiles
};

function replaceResources(url, html, callback) {
  var $ = cheerio.load(html);
  var selectors = Object.keys(bundleable);
  var functions = [];
  for (var i = 0, len = selectors.length; i < len; ++i) {
    // A closure over `selector` is created here so that the function
    // created for async.series has a reference to the selector at the
    // appropriate part of the loop, not just the last one.
    functions.push(function (selector) {
      return function (callback) {
        console.log('Looking for items with selector ' + selector);
        bundleable[selector]($, selector, url, callback);
      };
    }(selectors[i]));
  }
  async.series(functions, function (err, cheerios) {
    if (err) {
      callback(err, null);
    } else {
      console.log('Finished calling series of handlers');
      // The call to series will produce an array of Cheerio objects.
      // We probably need a way to merge the changes but, for now,
      // we'll just produce the last one.
      callback(null, cheerios[cheerios.length - 1].html());
    }
  });
}

function dataURI(url, content) {
  var encoded = content.toString('base64');
  return 'data:' + mime.lookup(url) + ';base64,' + encoded;
}

function fetchAndReplace(attr, elem, url, callback) {
  var resource = elem.attr(attr);
  // For some reason top-level pages might make it here
  // and we want to break the function before trying to fetch them.
  if (typeof resource === 'undefined' || !resource) {
    return;
  }
  url = urllib.resolve(url, resource);
  request.get(url).end(function (err, result) {
    if (!err) {
      var newuri = dataURI(url, result.body);
      console.log('Computed data uri ' + newuri);
      elem.attr(attr, newuri);
      console.log('Replaced URL');
      callback(null, elem);
    } else {
      // Here, the callback is actually the function that continues
      // iterating in async.reduce, so it is imperitive that we call it.
      callback(err, null);
    }
  });
}

function replaceAll($, selector, url, attr, callback) {
  var elements = [];
  $(selector).each(function (index, elem) {
    var $_this = $(this);
    elements.push($_this);
  });
  async.reduce(elements, elements[0], function (memo, item, next) {
    console.log('Reducing ' + elements.length + ' elements');
    if (typeof memo.attr(attr) === 'undefined') {
      console.log('Skipping element');
      // In the case that we get something like a <script> tag with no
      // source or href to fetch, just skip it.
      // This might not be quite right, but we'll come back to it.
      next(null, item);
    } else {
      console.log('Processing new element');
      fetchAndReplace(attr, memo, url, next);
    }
  }, callback);
}

/***********************
 ** Handler Functions **
 ***********************/

function replaceImages($, selector, url, callback) {
  replaceAll($, selector, url, 'src', callback);
}

{'fraction_non_alphanumeric': 0.15123614663256607, 'fraction_numerical': 0.0023870417732310316, 'mean_word_length': 3.3580980683506687, 'pattern_counts': {'":': 0, '<': 6, '<?xml version=': 0, '>': 4, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6853076', 'n_tokens_mistral': 1775, 'n_tokens_neox': 1749, 'n_words': 551}