<commit_before>/**
 * Module dependencies
 */
var request = require('superagent')
  , fs = require('fs')
  , cheerio = require('cheerio')
  , _ = require('underscore');

/**
 * Rasp
 */
var Rasp = module.exports = function Rasp (opts) {
  if (!(this instanceof Rasp)) return new Rasp (opts);

  opts = opts || {};
};

Rasp.prototype._sensors = [];

Rasp.prototype.sense = function (sensor) {
  sensor = _.isArray(sensor) ? sensor : _.toArray(arguments);
  return this._sensors = this._sensors.concat(sensor), this;
};

Rasp.prototype.scrape = function (src, content, cb) {
  this._fetch(src, function (err, test) {
    if (err) return cb(err);
    return this._run(text, cb);
  }.bind(this));
};

Rasp.prototype._fetch = function (src, cb) {
  if (/http/.test(src)) {
    request
      .get(src)
      .end(function (err, res) {
        return cb(err, res.text)
      });
  } else if (fs.existsSync(src)) {
    fs.readFile(src, 'utf8', cb);
  } else {
    cb(null, src);
  }

  return this;
};

Rasp.prototype._run = function (text, cb) {
  var index = 0
    , done = false
    , result = {}
    , sensors = this._sensors
    , selector = this._content
    , res = {}
    , error;

  // Wrap document in cheerio selector
  var $ = cheerio.load(text);

  // Build doc object
  var doc = { text: text, $: $, content: selector };

  // Build result object
  var res = {};

  // Bootstrap
  next.call(this);

  return this;

  function next (err) {
    // Got an error, shut it down
    if (err) return done = true, cb(err);
    // Already finished somewhere, float away
    if (done) return;

    // Grab next sensor
    var sensor = sensors[index++];

    // No sensors left, callback with the result
    if (!sensor) return done = true, cb(null, res);

    // Run the next sensor
    sensor.call(this, doc, res, _.once(next));
  }
};
<commit_msg>Correct text<commit_after>/**
 * Module dependencies
 */
var request = require('superagent')
  , fs = require('fs')
  , cheerio = require('cheerio')
  , _ = require('underscore');

/**
 * Rasp
 */
var Rasp = module.exports = function Rasp (opts) {
  if (!(this instanceof Rasp)) return new Rasp (opts);

  opts = opts || {};
};

Rasp.prototype._sensors = [];

Rasp.prototype.sense = function (sensor) {
  sensor = _.isArray(sensor) ? sensor : _.toArray(arguments);
  return this._sensors = this._sensors.concat(sensor), this;
};

Rasp.prototype.scrape = function (src, content, cb) {
  this._fetch(src, function (err, text) {
    if (err) return cb(err);
    return this._run(text, cb);
  }.bind(this));
};

Rasp.prototype._fetch = function (src, cb) {
  if (/http/.test(src)) {
    request
      .get(src)
      .end(function (err, res) {
        return cb(err, res.text)
      });
  } else if (fs.existsSync(src)) {
    fs.readFile(src, 'utf8', cb);
  } else {
    cb(null, src);
  }

  return this;
};

Rasp.prototype._run = function (text, cb) {
  var index = 0
    , done = false
    , result = {}
    , sensors = this._sensors
    , selector = this._content
    , res = {}
    , error;

  // Wrap document in cheerio selector
  var $ = cheerio.load(text);

  // Build doc object
  var doc = { text: text, $: $, content: selector };

  // Build result object
  var res = {};

  // Bootstrap
  next.call(this);

  return this;

  function next (err) {
    // Got an error, shut it down
    if (err) return done = true, cb(err);
    // Already finished somewhere, float away
    if (done) return;

    // Grab next sensor
    var sensor = sensors[index++];

    // No sensors left, callback with the result
    if (!sensor) return done = true, cb(null, res);

    // Run the next sensor
    sensor.call(this, doc, res, _.once(next));
  }
};

{'fraction_non_alphanumeric': 0.16047709406343183, 'fraction_numerical': 0.0010843046896177825, 'mean_word_length': 2.7122736418511066, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6824189', 'n_tokens_mistral': 1259, 'n_tokens_neox': 1213, 'n_words': 349}