<commit_before>/**
 * Modules dependencies
 */

var mocha = require('mocha'),
	assert = require('chai').assert,
	libPath = process.env['SCRAPINODE_COV'] ? '../lib-cov' : '../lib',
	scrapinode = require( libPath + '/scrapinode'),
	ScrapinodeError = require(libPath + '/error/scrapinode-error'),
	express = require('express'),
	filed = require('filed');

// Web app
var app = express();

app.get('/page-open-graph.html',function(req,res){
	req.pipe(filed(__dirname + '/resources/page-open-graph.html')).pipe(res);
});

app.get('/page-schema-org.html',function(req,res){
	req.pipe(filed(__dirname + '/resources/page-schema-org.html')).pipe(res);
});

app.get('/page-generic-tags.html',function(req,res){
	req.pipe(filed(__dirname + '/resources/page-generic-tags.html')).pipe(res);
});

app.listen(1102);

// Test suite

['jsdom','cheerio'].forEach(runTestSuite);

function runTestSuite(engine){
	describe('scraper using the default operators thanks to ' + engine,function(){
		beforeEach(function(){
			scrapinode.clearRouter();
		});
		describe('and scraping by following the Open Graph protocol',function(){
			describe('#get("title")',function(){
				it('should retrieve the text representating the title',function(done){
					scrapinode.useAll(scrapinode.defaults());
					scrapinode.createScraper('http://localhost:1102/page-open-graph.html',function(err,scraper){
						assert.isNull(err);
						assert.equal(scraper.get('title'),'The Rock');
						done();
					});
				});
			});
			describe('#get("descriptions")',function(){
				it('should retrieve a list of text representating the descriptions',function(done){
					scrapinode.useAll(scrapinode.defaults());
					scrapinode.createScraper('http://localhost:1102/page-open-graph.html',function(err,scraper){
						assert.isNull(err);
						assert.deepEqual(scraper.get('descriptions'),['<NAME> found fame and fortune as the suave, sophisticated British agent, <NAME>']);
						done();
					});
				});
			});
			describe('#get("images")',function(){
				it('should retrieve a list of images url',function(done){
					scrapinode.useAll(scrapinode.defaults());
					scrapinode.createScraper('http://localhost:1102/page-open-graph.html',function(err,scraper){
						assert.isNull(err);
						assert.deepEqual(scraper.get('images'),['http://ia.media-imdb.com/images/rock.jpg']);
						done();
					});
				});
			});
			describe('#get("videos")',function(){
				it('should retrieve a list of videos [urls or html representations of the videos]',function(done){
					scrapinode.useAll(scrapinode.defaults());
					scrapinode.createScraper('http://localhost:1102/page-open-graph.html',function(err,scraper){
						assert.isNull(err);
						assert.deepEqual(scraper.get('videos'),['http://example.com/awesome.flv']);
						done();
					});
				});
			});
		});
		describe('and scraping by following the Schema.org specifications',function(){
			describe('#get("title")',function(){
				it('should retrieve the text representating the title',function(done){
					scrapinode.useAll(scrapinode.defaults());
					scrapinode.createScraper('http://localhost:1102/page-schema-org.html',function(err,scraper){
						assert.isNull(err);
						assert.equal(scraper.get('title'),'Call of Duty: Modern Warfare III');
						done();
					});
				});
			});
			describe('#get("descriptions")',function(){
				it('should retrieve a list of text representating the descriptions',function(done){
					scrapinode.useAll(scrapinode.defaults());
					scrapinode.createScraper('http://localhost:1102/page-schema-org.html',function(err,scraper){
						assert.isNull(err);
						assert.deepEqual(scraper.get('descriptions'),['First Person Shooter game available on consoles and PCs.']);
						done();
					});
				});
			});
			describe('#get("images")',function(){
				it('should retrieve a list of images url',function(done){
					scrapinode.useAll(scrapinode.defaults());
					scrapinode.createScraper('http://localhost:1102/page-schema-org.html',function(err,scraper){
						assert.isNull(err);
						assert.deepEqual(scraper.get('images'),['http://www.journaldugamer.com/files/2011/11/mw3-header.jpg']);
						done();
					});
				});
			});
		});
		describe('and scraping by searching the generic html tags',function(){
			describe('#get("title")',function(){
				it('should retrieve the text representating the title',function(done){
					scrapinode.useAll(scrapinode.defaults());
					scrapinode.createScraper('http://localhost:1102/page-generic-tags.html',function(err,scraper){
						assert.isNull(err);
						assert.equal(scraper.get('title'),'generic tags page');
						done();
					});
				});
			});
			describe('#get("descriptions")',function(){
				it('should retrieve a list of text representating the descriptions',function(done){
					scrapinode.useAll(scrapinode.defaults());
					scrapinode.createScraper('http://localhost:1102/page-generic-tags.html',function(err,scraper){
						assert.isNull(err);
						assert.deepEqual(scraper.get('descriptions'),['description of the generics tag page']);
						done();
					});
				});
			});
			describe('#get("images")',function(){
				it('should retrieve a list of images url',function(done){
					scrapinode.useAll(scrapinode.defaults());
					scrapinode.createScraper('http://localhost:1102/page-generic-tags.html',function(err,scraper){
						assert.isNull(err);
						assert.deepEqual(scraper.get('images'),['http://localhost/img1.png','http://localhost/img2.png']);
						done();
					});
				});
			});
			describe('#get("videos")',function(){
				it('should retrieve a list of videos [urls or html representations of the videos]',function(done){
					scrapinode.useAll(scrapinode.defaults());
					scrapinode.createScraper('http://localhost:1102/page-generic-tags.html',function(err,scraper){
						assert.isNull(err);
						assert.deepEqual(scraper.get('videos'),['http://localhost:1102/videofile.ogg','http://localhost:1102/movie.mov']);
						done();
					});
				});
			});
		});
	});
}
<commit_msg>NEW: Use the engine given in argument.<commit_after>/**
 * Modules dependencies
 */

var mocha = require('mocha'),
	assert = require('chai').assert,
	libPath = process.env['SCRAPINODE_COV'] ? '../lib-cov' : '../lib',
	scrapinode = require( libPath + '/scrapinode'),
	ScrapinodeError = require(libPath + '/error/scrapinode-error'),
	express = require('express'),
	filed = require('filed');

// Web app
var app = express();

app.get('/page-open-graph.html',function(req,res){
	req.pipe(filed(__dirname + '/resources/page-open-graph.html')).pipe(res);
});

app.get('/page-schema-org.html',function(req,res){
	req.pipe(filed(__dirname + '/resources/page-schema-org.html')).pipe(res);
});

app.get('/page-generic-tags.html',function(req,res){
	req.pipe(filed(__dirname + '/resources/page-generic-tags.html')).pipe(res);
});

app.listen(1102);

// Test suite

['jsdom','cheerio'].forEach(runTestSuite);

function runTestSuite(engine){
	describe('scraper using the default operators thanks to ' + engine,function(){
		beforeEach(function(){
			scrapinode.clearRouter();
		});
		describe('and scraping by following the Open Graph protocol',function(){
			describe('#get("title")',function(){
				it('should retrieve the text representating the title',function(done){
					scrapinode.useAll(scrapinode.defaults());
					var options = {
						url : 'http://localhost:1102/page-open-graph.html',
						engine : engine
					};
					scrapinode.createScraper(options,function(err,scraper){
						assert.isNull(err);
						assert.equal(scraper.get('title'),'The Rock');
						done();
					});
				});
			});
			describe('#get("descriptions")',function(){
				it('should retrieve a list of text representating the descriptions',function(done){
					scrapinode.useAll(scrapinode.defaults());
					var options = {
						url : 'http://localhost:1102/page-open-graph.html',
						engine : engine
					};
					scrapinode.createScraper(options,function(err,scraper){
						assert.isNull(err);
						assert.deepEqual(scraper.get('descriptions'),['<NAME> found fame and fortune as the suave, sophisticated British agent, <NAME>']);
						done();
					});
				});
			});
			describe('#get("images")',function(){
				it('should retrieve a list of images url',function(done){
					scrapinode.useAll(scrapinode.defaults());
					var options = {
						url : 'http://localhost:1102/page-open-graph.html',
						engine : engine
					};
					scrapinode.createScraper(options,function(err,scraper){
						assert.isNull(err);
						assert.deepEqual(scraper.get('images'),['http://ia.media-imdb.com/images/rock.jpg']);
						done();
					});
				});
			});
			describe('#get("videos")',function(){
				it('should retrieve a list of videos [urls or html representations of the videos]',function(done){
					scrapinode.useAll(scrapinode.defaults());
					var options = {
						url : 'http://localhost:1102/page-open-graph.html',
						engine : engine
					};
					scrapinode.createScraper(options,function(err,scraper){
						assert.isNull(err);
						assert.deepEqual(scraper.get('videos'),['http://example.com/awesome.flv']);
						done();
					});
				});
			});
		});
		describe('and scraping by following the Schema.org specifications',function(){
			describe('#get("title")',function(){
				it('should retrieve the text representating the title',function(done){
					scrapinode.useAll(scrapinode.defaults());
					var options = {
						url : 'http://localhost:1102/page-schema-org.html',
						engine : engine
					};
					scrapinode.createScraper(options,function(err,scraper){
						assert.isNull(err);
						assert.equal(scraper.get('title'),'Call of Duty: Modern Warfare III');
						done();
					});
				});
			});
			describe('#get("descriptions")',function(){
				it('should retrieve a list of text representating the descriptions',function(done){
					scrapinode.useAll(scrapinode.defaults());
					var options = {
						url : 'http://localhost:1102/page-schema-org.html',
						engine : engine
					};
					scrapinode.createScraper(options,function(err,scraper){
						assert.isNull(err);
						assert.deepEqual(scraper.get('descriptions'),['First Person Shooter game available on consoles and PCs.']);
						done();
					});
				});
			});
			describe('#get("images")',function(){
				it('should retrieve a list of images url',function(done){
					scrapinode.useAll(scrapinode.defaults());
					var options = {
						url : 'http://localhost:1102/page-schema-org.html',
						engine : engine
					};
					scrapinode.createScraper(options,function(err,scraper){
						assert.isNull(err);
						assert.deepEqual(scraper.get('images'),['http://www.journaldugamer.com/files/2011/11/mw3-header.jpg']);
						done();
					});
				});
			});
		});
		describe('and scraping by searching the generic html tags',function(){
			describe('#get("title")',function(){
				it('should retrieve the text representating the title',function(done){
					scrapinode.useAll(scrapinode.defaults());
					scrapinode.createScraper('http://localhost:1102/page-generic-tags.html',function(err,scraper){
						assert.isNull(err);
						assert.equal(scraper.get('title'),'generic tags page');
						done();
					});
				});
			});
			describe('#get("descriptions")',function(){
				it('should retrieve a list of text representating the descriptions',function(done){
					scrapinode.useAll(scrapinode.defaults());
					var options = {
						url : 'http://localhost:1102/page-generic-tags.html',
						engine : engine
					};
					scrapinode.createScraper(options,function(err,scraper){
						assert.isNull(err);
						assert.deepEqual(scraper.get('descriptions'),['description of the generics tag page']);
						done();
					});
				});
			});
			describe('#get("images")',function(){
				it('should retrieve a list of images url',function(done){
					scrapinode.useAll(scrapinode.defaults());
					var options = {
						url : 'http://localhost:1102/page-generic-tags.html',
						engine : engine
					};
					scrapinode.createScraper(options,function(err,scraper){
						assert.isNull(err);
						assert.deepEqual(scraper.get('images'),['http://localhost/img1.png','http://localhost/img2.png']);
						done();
					});
				});
			});
			describe('#get("videos")',function(){
				it('should retrieve a list of videos [urls or html representations of the videos]',function(done){
					scrapinode.useAll(scrapinode.defaults());
					var options = {
						url : 'http://localhost:1102/page-generic-tags.html',
						engine : engine
					};
					scrapinode.createScraper(options,function(err,scraper){
						assert.isNull(err);
						assert.deepEqual(scraper.get('videos'),['http://localhost:1102/videofile.ogg','http://localhost:1102/movie.mov']);
						done();
					});
				});
			});
		});
	});
}

{'fraction_non_alphanumeric': 0.18049125009809308, 'fraction_numerical': 0.010201679353370479, 'mean_word_length': 3.9395348837209303, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3712261', 'n_tokens_mistral': 5068, 'n_tokens_neox': 3927, 'n_words': 604}