<commit_before>var promisify = require('deferred').promisify;
var Tasker = require('tasker');
var fs = require('fs');

/*
 * grunt-scraper
 * https://github.com/sammerry/scraper
 *
 * Copyright (c) 2014 <NAME>
 * Licensed under the MIT license.
 */

'use strict';

module.exports = function(grunt) {

  var jobList = {};
  var jobsDIR = [process.cwd(),grunt.config.get('tasker.tasks_file')].join('/');
  var jobsDescription = 'A custom task from the '
                      + grunt.config.get('scraper.jobs')
                      + ' folder';


  /*
   *
   * Build task list.
   *
   */
  var files = fs.readdirSync(jobsDIR);
  var len = files.length;

  while (len--) {
    var task = files[len];

    // TODO: handle sub-folders.
    if (/\.js$/.test(task)) {
      var taskMatch = task.match(/(.*)\.js$/) || [];
      var taskName = taskMatch[1];
      var taskPath = jobsDIR+'/'+taskName+'.js';

      jobList[taskName] = taskPath;
      grunt.registerTask(taskName, jobsDescription, taskRunner);
    }
  }




  /*
   *
   * Create runable tasks from list.
   *
   */
  function taskRunner () {
    var done = this.async();
    var jobPath = jobList[this.name];
    var options = grunt.config.get('tasker.'+'this.name')||{};
    options.path = jobPath;
    return new Tasker(options, done);
  };




  // list all available jobs
  grunt.registerTask('tasker', 'list available jobs', function () {
    console.log('All Available Jobs:');
    for (var name in jobList) {
      console.log('   '+name+'   ');
    }
  });
};<commit_msg>fixing options<commit_after>var promisify = require('deferred').promisify;
var Tasker = require('tasker');
var fs = require('fs');

/*
 * grunt-scraper
 * https://github.com/sammerry/scraper
 *
 * Copyright (c) 2014 <NAME>
 * Licensed under the MIT license.
 */

'use strict';

module.exports = function(grunt) {

  var jobList = {};
  var jobsDIR = [process.cwd(),grunt.config.get('tasker.tasks_file')].join('/');
  var jobsDescription = 'A custom task from the '
                      + grunt.config.get('scraper.jobs')
                      + ' folder';


  /*
   *
   * Build task list.
   *
   */
  var files = fs.readdirSync(jobsDIR);
  var len = files.length;

  while (len--) {
    var task = files[len];

    // TODO: handle sub-folders.
    if (/\.js$/.test(task)) {
      var taskMatch = task.match(/(.*)\.js$/) || [];
      var taskName = taskMatch[1];
      var taskPath = jobsDIR+'/'+taskName+'.js';

      jobList[taskName] = taskPath;
      grunt.registerTask(taskName, jobsDescription, taskRunner);
    }
  }




  /*
   *
   * Create runable tasks from list.
   *
   */
  function taskRunner () {
    var done = this.async();
    var jobPath = jobList[this.name];
    var options = grunt.config.get('tasker.'+ this.name)||{};
    options.path = jobPath;
    return new Tasker(options, done);
  };




  // list all available jobs
  grunt.registerTask('tasker', 'list available jobs', function () {
    console.log('All Available Jobs:');
    for (var name in jobList) {
      console.log('   '+name+'   ');
    }
  });
};
{'fraction_non_alphanumeric': 0.17024041585445093, 'fraction_numerical': 0.003248862897985705, 'mean_word_length': 2.510832383124287, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7101034', 'n_tokens_mistral': 1071, 'n_tokens_neox': 1021, 'n_words': 226}