<commit_before>var async = require('async');
var cheerio = require('cheerio');
var log4js = require('log4js');
var request = require('request');
var EventEmitter = require('events');
var util = require('util');

var Scraper = exports.Scraper = (function(){
  var logger = log4js.getLogger("scraper");

  var Class = function (url){
    this.url = url;
    EventEmitter.call(this);
    this.on("item", this.pipe.bind(this));
  };
  util.inherits(Class, EventEmitter);
  var proto = Class.prototype;

  proto.run = function(cb) {
    var self = this;
    self.get(function($){
      self.scrape($, cb);
    });
  };

  proto.scrape = function($, cb) {
    this.emit("item", "default scraper called");
  };

  proto.pipe = function(item) {
    console.log("got item:", item);
  };

  proto.get = function(cb){
    var url = this.url;
    logger.debug("getting", url);
    request(url, function (err, res, body) {
      if (!err && res.statusCode == 200) {
        cb(cheerio.load(body));
      } else {
        var code = (res && res.statusCode) || "-"
        logger.error("getting", url, code, err);
      }
    });
  };

  return Class;
})();

exports.crawl = function(scraper, concurrency, cb) {
  var q = async.queue(function(scraper, cb) {
    scraper.run(function(scrapers){
      if (scrapers && scrapers.length) {
        scrapers.forEach(function(s){
          q.push(s);
        });
        cb();
      } else {
        cb();
      }
    });
  }, concurrency);

  q.drain = cb;
  q.push(scraper);
};

exports.defineScraper = function(object, base) {
  if (!base) base = Scraper;
  var Class = function(){
    var url = arguments[0];
    var args = [];
    for(var i=0; i<arguments.length; i++) {
      args.push(arguments[i]);
    }
    if (object.init) {
      object.init.apply(this, args);
    }
    base.call(this, url);
  };
  util.inherits(Class, base);

  for (var k in object) {
    if (k !== "init" && typeof(object[k]) === "function") {
      Class.prototype[k] = object[k];
    }
  }

  return Class;
}

<commit_msg>better interface<commit_after>var async = require('async');
var cheerio = require('cheerio');
var log4js = require('log4js');
var request = require('request');
var EventEmitter = require('events');
var util = require('util');

var emitter = new EventEmitter();
exports.emitItem = function(item, scraper) {
  emitter.emit("item", item, scraper);
};

exports.emitScraper = function(scraper) {
  emitter.emit("scraper", scraper);
};

exports.onItem = function(cb) {
  emitter.on("item", cb);
};

var Scraper = exports.Scraper = (function(){
  var logger = log4js.getLogger("scraper");

  var Class = function (url){
    this.url = url;
  };

  var proto = Class.prototype;

  proto.run = function(cb) {
    var self = this;
    var scrapers = [];
    self.get(function($){
      self.scrape($, cb);
    });
  };

  proto.scrape = function($, cb) {
    worm.emitItem("default scraper called");
  };

  proto.get = function(cb){
    var url = this.url;
    logger.debug("getting", url);
    request(url, function (err, res, body) {
      if (!err && res.statusCode == 200) {
        cb(cheerio.load(body));
      } else {
        var code = (res && res.statusCode) || "-"
        logger.error("getting", url, code, err);
      }
    });
  };

  return Class;
})();

exports.crawl = function(scraper, concurrency, cb) {
  var q = async.queue(function(scraper, cb) {
    scraper.run(function(){
      cb();
    });
  }, concurrency);

  q.drain = cb;
  emitter.on("scraper", function(scraper){
    q.push(scraper);
  });

  q.push(scraper);
};

exports.defineScraper = function(object, base) {
  if (!base) base = Scraper;
  var Class = function(){
    var url = arguments[0];
    var args = [];
    for(var i=0; i<arguments.length; i++) {
      args.push(arguments[i]);
    }
    if (object.init) {
      object.init.apply(this, args);
    }
    base.call(this, url);
  };
  util.inherits(Class, base);

  for (var k in object) {
    if (k !== "init" && typeof(object[k]) === "function") {
      Class.prototype[k] = object[k];
    }
  }

  return Class;
}


{'fraction_non_alphanumeric': 0.18277465133349646, 'fraction_numerical': 0.003914851969659897, 'mean_word_length': 2.882241215574549, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3324877', 'n_tokens_mistral': 1458, 'n_tokens_neox': 1423, 'n_words': 306}