<commit_before>var Nightmare1 = require('nightmare');
var nightmare = Nightmare1({
  show: false, // true
  'proxy-server': '192.168.3.11:5678',
  'ignore-certificate-errors': true
})
  .useragent("Mozilla/5.0 (Windows NT 6.3; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/38.0.2125.111 Safari/537.36");
var fs = require('fs');
var URL = require('url');
var sanitizeFilename = require("sanitize-filename");
var mkdirp = require('mkdirp');
var _ = require('lodash');

var DEFAULT_OPTIONS = {
  outputDirectory: 'out',
  querySelector: 'html'
};


var parseAsUrls = function (links) {
  return (links || []).map(link => {
    /**
     * URL e.g.:
     * auth = null
     * hash = null
     * hostname = "blog.fefe.de"
     * href = "http://blog.fefe.de/?ts=a6762575"
     * path = "/?ts=a6762575"
     * pathname = "/"
     * port = null
     * protocol = "http:"
     * query = "ts=a6762575"
     * search = "?ts=a6762575"
     */
    return URL.parse(link);
  }).filter(link => !!link);
};
/**
 * grepit
 * Description
 *
 * @name grepit
 * @function
 * @param {Array} links An array of links
 * @param {Object} options An object containing the following fields:
 *
 * @return {function} Result
 */

module.exports = function (links, options) {
  var _links = parseAsUrls(links);

  var _options = _.defaults(options, DEFAULT_OPTIONS);

  var querySelector = _options.querySelector;
  var outputDirectory = sanitizeFilename(_options.outputDirectory);
  mkdirp.sync(outputDirectory);

  return function *() {
    //run the page HTML retrieval and save for each link
    for (var idx in _links) {
      var link = _links[idx];
      console.log('fetching ' + link.href);

      var filename = sanitizeFilename(link.href + '.txt');
      var hostOutputDirectory = outputDirectory + '/' + sanitizeFilename(link.host);
      mkdirp.sync(hostOutputDirectory);

      var out = hostOutputDirectory + '/' + filename;

      var page = yield nightmare.goto(link.href)
        .evaluate(function (link, querySelector) {
          var html = document.querySelector(querySelector);
          if (!html) {
            return '## grepit: no ' + querySelector + ' tag found';
          }
          return html.innerHTML;
        }, link, querySelector);

      fs.writeFileSync(out, page);
      console.log('successfully written to file ' + out);
    }
    yield nightmare.end();
  };
};<commit_msg>skip links if file is already present<commit_after>var Nightmare1 = require('nightmare');
var nightmare = Nightmare1({
  show: false, // true
  'proxy-server': '192.168.3.11:5678',
  'ignore-certificate-errors': true
})
  .useragent("Mozilla/5.0 (Windows NT 6.3; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/38.0.2125.111 Safari/537.36");
var fs = require('fs');
var URL = require('url');
var sanitizeFilename = require("sanitize-filename");
var mkdirp = require('mkdirp');
var _ = require('lodash');

var DEFAULT_OPTIONS = {
  outputDirectory: 'out',
  querySelector: 'html'
};


var parseAsUrls = function (links) {
  return (links || []).map(link => {
    /**
     * URL e.g.:
     * auth = null
     * hash = null
     * hostname = "blog.fefe.de"
     * href = "http://blog.fefe.de/?ts=a6762575"
     * path = "/?ts=a6762575"
     * pathname = "/"
     * port = null
     * protocol = "http:"
     * query = "ts=a6762575"
     * search = "?ts=a6762575"
     */
    return URL.parse(link);
  }).filter(link => !!link);
};
/**
 * grepit
 * Description
 *
 * @name grepit
 * @function
 * @param {Array} links An array of links
 * @param {Object} options An object containing the following fields:
 *
 * @return {function} Result
 */

module.exports = function (links, options) {
  var _links = parseAsUrls(links);

  var _options = _.defaults(options, DEFAULT_OPTIONS);

  var querySelector = _options.querySelector;
  var outputDirectory = sanitizeFilename(_options.outputDirectory);
  mkdirp.sync(outputDirectory);

  return function *() {
    //run the page HTML retrieval and save for each link
    for (var idx in _links) {
      var link = _links[idx];

      var filename = sanitizeFilename(link.href + '.txt');
      var hostOutputDirectory = outputDirectory + '/' + sanitizeFilename(link.host);
      mkdirp.sync(hostOutputDirectory);

      var out = hostOutputDirectory + '/' + filename;

      if (fs.existsSync(out)) {
        console.log('skipping ' + link.href + ' as it already exists.');
      } else {
        console.log('fetching ' + link.href);

        var page = yield nightmare.goto(link.href)
          .evaluate(function (link, querySelector) {
            var html = document.querySelector(querySelector);
            if (!html) {
              return '## grepit: no ' + querySelector + ' tag found';
            }
            return html.innerHTML;
          }, link, querySelector);

        fs.writeFileSync(out, page);
        console.log('successfully written to file ' + out);
      }
    }
    yield nightmare.end();
  };
};
{'fraction_non_alphanumeric': 0.14740875176446863, 'fraction_numerical': 0.027828191167574106, 'mean_word_length': 2.914759273875296, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 2, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1819847', 'n_tokens_mistral': 1650, 'n_tokens_neox': 1545, 'n_words': 383}