<commit_before>/*!
 * grunt-fetch-pages
 * https://github.com/olihel/grunt-fetch-pages.git
 *
 * Copyright (c) 2013-2014 <NAME>
 * Released under MIT license (https://raw.github.com/olihel/grunt-fetch-pages/master/LICENSE-MIT)
 */

module.exports = function (grunt) {
  'use strict';

  var fetchPages = require('./lib/fetchpages');
  var getPagesFromFiles = require('./lib/getPagesFromFiles');
  var getPagesFromURLs = require('./lib/getPagesFromURLs');

  var url = require('url');

  grunt.registerMultiTask('fetchpages', 'Fetch URLs and save the result as local files', function () {
    var done = this.async();
    var pages = [];
    var options = this.options({
      'urls': [],
      'followLinks': true,
      'ignoreSelector': '[rel="nofollow"]'
    });

    var destinationFolder = options.destinationFolder;
    var baseURL = options.baseURL;
    var urls = options.urls;
    var followLinks = options.followLinks;
    var ignoreSelector = options.ignoreSelector;

    if (!destinationFolder) {
      grunt.log.error('"destinationFolder" option is mandatory!');
      done(false);
    }<commit_msg>upper case char<commit_after>/*!
 * grunt-fetch-pages
 * https://github.com/olihel/grunt-fetch-pages.git
 *
 * Copyright (c) 2013-2014 <NAME>
 * Released under MIT license (https://raw.github.com/olihel/grunt-fetch-pages/master/LICENSE-MIT)
 */

module.exports = function (grunt) {
  'use strict';

  var fetchPages = require('./lib/fetchPages');
  var getPagesFromFiles = require('./lib/getPagesFromFiles');
  var getPagesFromURLs = require('./lib/getPagesFromURLs');

  var url = require('url');

  grunt.registerMultiTask('fetchpages', 'Fetch URLs and save the result as local files', function () {
    var done = this.async();
    var pages = [];
    var options = this.options({
      'urls': [],
      'followLinks': true,
      'ignoreSelector': '[rel="nofollow"]'
    });

    var destinationFolder = options.destinationFolder;
    var baseURL = options.baseURL;
    var urls = options.urls;
    var followLinks = options.followLinks;
    var ignoreSelector = options.ignoreSelector;

    if (!destinationFolder) {
      grunt.log.error('"destinationFolder" option is mandatory!');
      done(false);
    }
{'fraction_non_alphanumeric': 0.14734950584007186, 'fraction_numerical': 0.0071877807726864335, 'mean_word_length': 3.554192229038855, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 4, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7206803', 'n_tokens_mistral': 689, 'n_tokens_neox': 665, 'n_words': 152}