<commit_before>/*!
 * scrapinode
 * Copyright(c) 2013 <NAME>
 * Author <NAME> <<EMAIL>>
 * MIT Licensed
 */

/**
 * Modules dependencies
 */

var browser = require('./browser'),
  Router = require('./router'),
  Route = require('./route'),
  Scraper = require('./scraper'),
  defaults = require('./defaults/');

// Expose scrapinode
var scrapinode = exports;

// Create the main router
var router = new Router();

/**
 * Expose engine used to build the DOM
 *  (available: "jsdom" or "cheerio")
 */

scrapinode.engine = 'jsdom';

/**
 * Create a scraper for the given `url` with the given `engine`
 *
 * @param {String|Object} options - url of the page or a set of options in this case options.url is mandatory
 * @param {Function} callback -
 * @param {Error} callback().err -
 * @param {Scraper} callback().scraper -
 *
 * @api public
 */

scrapinode.createScraper = function (options,callback){
    if(typeof(options) === 'string'){
      var url = options;
      options = {
        url : url,
        engine : scrapinode.engine
      };
    }

    /* TODO re-enable HTML feature
    if(url.match(/^https?:/)){
      options.url = url;
    }else{
      options.html = url;
    }
    */

    browser.load(options,function(err,window){
      if(err) return callback(err);
      var scraper = new Scraper(options.url,window,router);
      callback(err,scraper);
    });
};

/**
 * Add the given `operation` in the router for the given `path` and `content`
 *
 * @param {String|RegExp} path -
 * @param {String|RegExp} content -
 * @param {Function} operation -
 *
 * @return {this}
 * @api public
 */

scrapinode.use = function(path,content,operation){
  router.addRoute(new Route(path,content,operation));
  return scrapinode;
};

/**
 * Add all given `routes` caracterized by a `path`, a `content` and an `operation` in the `router`
 *
 * @param {Array} routes -
 * @param {String|RegExp} routes[].path -
 * @param {String|RegExp} routes[].content -
 * @param {Function} routes[].operation -
 *
 * @return {this}
 * @api public
 */

scrapinode.useAll = function(routes){
  routes.forEach(function(route){
    router.addRoute(new Route(route.path,route.content,route.operation));
  });
  return scrapinode;
};

/**
 * Expose default routes for the contents "title", "descriptions", "videos", "images"
 *
 * @return {Array} routes
 * @api public
 */

scrapinode.defaults = function(){
  return defaults;
};


/**
 * Remove all the routes available in the router
 *
 * @return
 * @api public
 */

scrapinode.clearRouter = function(){
  router.routes = [];
};<commit_msg>NEW: Allow to create multiple instance of scrapinode.<commit_after>/*!
 * scrapinode
 * Copyright(c) 2013 <NAME>
 * Author <NAME> <<EMAIL>>
 * MIT Licensed
 */

/**
 * Modules dependencies
 */

var browser = require('./browser'),
  Router = require('./router'),
  Route = require('./route'),
  Scraper = require('./scraper'),
  defaults = require('./defaults/');

/**
 * Expose an instance of `Scrapinode`
 *  as a convenience mostly
 */

module.exports = new Scrapinode();

/**
 * Create an instance of ̀Scrapinode`
 *
 * @return {Scrapinode}
 * @api public
 */

exports.createInstance = function(engine){
  return new Scrapinode(engine);
};

/**
 * `Scrapinode` class
 *
 * @constructor
 * @api private
 */

function Scrapinode(engine){
  this.engine = engine || 'jsdom';
  this.router = new Router();
};

/**
 * Create a scraper for the given `url` with the given `engine`
 *
 * @param {String|Object} options - url of the page or a set of options in this case options.url is mandatory
 * @param {Function} callback -
 * @param {Error} callback().err -
 * @param {Scraper} callback().scraper -
 *
 * @api public
 */

Scrapinode.prototype.createScraper = function (options,callback){
  var self = this;
  if(typeof(options) === 'string'){
    var url = options;
    options = {
      url : url,
      engine : this.engine
    };
  }

  /* TODO re-enable HTML feature
  if(url.match(/^https?:/)){
    options.url = url;
  }else{
    options.html = url;
  }
  */

  browser.load(options,function(err,window){
    if(err) return callback(err);
    var scraper = new Scraper(options.url,window,self.router);
    callback(err,scraper);
  });
};

/**
 * Add the given `operation` in the router for the given `path` and `content`
 *
 * @param {String|RegExp} path -
 * @param {String|RegExp} content -
 * @param {Function} operation -
 *
 * @return {this}
 * @api public
 */

Scrapinode.prototype.use = function(path,content,operation){
  this.router.addRoute(new Route(path,content,operation));
  return this;
};

/**
 * Add all given `routes` caracterized by a `path`, a `content` and an `operation` in the `router`
 *
 * @param {Array} routes -
 * @param {String|RegExp} routes[].path -
 * @param {String|RegExp} routes[].content -
 * @param {Function} routes[].operation -
 *
 * @return {this}
 * @api public
 */

Scrapinode.prototype.useAll = function(routes){
  var self = this;
  routes.forEach(function(route){
    self.router.addRoute(new Route(route.path,route.content,route.operation));
  });
  return self;
};


/**
 * Remove all the routes available in the router
 *
 * @return
 * @api public
 */

Scrapinode.prototype.clearRouter = function(){
  this.router.routes = [];
  return this;
};

/**
 * Expose default routes for the contents "title", "descriptions", "videos", "images"
 *
 * @return {Array} routes
 * @api public
 */

Scrapinode.prototype.defaults = function(){
  return defaults;
};
{'fraction_non_alphanumeric': 0.16115475972958157, 'fraction_numerical': 0.0014617211766855473, 'mean_word_length': 3.4431818181818183, 'pattern_counts': {'":': 0, '<': 11, '<?xml version=': 0, '>': 11, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6465162', 'n_tokens_mistral': 1758, 'n_tokens_neox': 1750, 'n_words': 478}