<commit_before>/** 
 * Modules dependencies
 */

var vows = require('vows'),
    assert = require('assert'),
    scrapinode = require('./../main');
    Scraper = require('./../lib/actors/scraper')
// Vows test suite
var suite = vows.describe('Scrapinode');

// Elements for the test suite
var pathScrapulesDir = __dirname + '/scrapules/';
var urlDefault = 'http://www.comet.co.uk/p/Plasma-TVs/buy-LG-42PT353-Plasma-TV/721050';
var urlCultBeauty = 'http://www.cultbeauty.co.uk/clarisonic/clarisonic-plus_sonic_skin_cleansing_system.php';

suite.addBatch({
   'Load a scrapule in scrapinode' : {
      topic : function(){
         var useScrapules = function(){ scrapinode.use(pathScrapulesDir)}
         return useScrapules
      },
      'should happen without throwing errors' : function(useScrapules){
         assert.doesNotThrow(useScrapules,Error)
      }
   }
}).addBatch({
   'Create a scraper' : {
      'to scrap an HTML page thanks to its url' : {
         topic : function(){
            scrapinode.init();
            scrapinode.createScraper(urlDefault,this.callback);
         },
         'should happen without error' : function(err,scraper){
            assert.isNull(err);
         },
         'should give a Scraper object': function(err,scraper){
            assert.instanceOf(scraper,Scraper);
         },
         'should give a Scraper object able to retrieve the title of the page' : function(err,scraper){
            assert.isString(scraper.get('title'));
         },
         'should give a Scraper object able to retrieve a description of the page' : function(err,scraper){
            assert.isArray(scraper.get('description'));
         },
         'should give a Scraper object able to retrieve images of the page' : function(err,scraper){
            assert.isArray(scraper.get('images'));
         }
      },
      'to scrap an HTML page thanks to its url using a specific scrapule determined by the domain name of the page' : {
         topic : function(){
            scrapinode.use(pathScrapulesDir);
            scrapinode.createScraper(urlCultBeauty,this.callback)
         },
         'should happen without error' : function(err,scraper){
            assert.isNull(err);
         },
         'should give a Scraper object' : function(err,scraper){
            assert.instanceOf(scraper,Scraper);
         },
         'should give a Scraper object able to retrieve the title of the page' : function(err,scraper){
            assert.isString(scraper.get('title'));
         },
         'should give a Scraper object able to retrieve a description of the page' : function(err,scraper){
            assert.isArray(scraper.get('description'));
         },
         'should give a Scraper object able to retrieve images of the page' : function(err,scraper){
            assert.isArray(scraper.get('images'));
         },
         'should give a Scraper object able to retrieve the price of the product' : function(err,scraper){
            assert.match(scraper.get('price'),/([0-9,]{1,}(\.?[0-9]{1,}))/)
         }
      }
   }
   
}).export(module)
<commit_msg>NEW: Add two tests.<commit_after>/** 
 * Modules dependencies
 */

var vows = require('vows'),
    assert = require('assert'),
    scrapinode = require('./../main');
    Scraper = require('./../lib/actors/scraper')
// Vows test suite
var suite = vows.describe('Scrapinode');

// Elements for the test suite
var pathScrapulesDir = __dirname + '/scrapules/';
var urlDefault = 'http://www.comet.co.uk/p/Plasma-TVs/buy-LG-42PT353-Plasma-TV/721050';
var urlCultBeauty = 'http://www.cultbeauty.co.uk/clarisonic/clarisonic-plus_sonic_skin_cleansing_system.php';
var urlImage = 'http://ecx.images-amazon.com/images/I/51RRbTkbfBL._SL500_AA300_.jpg';
var urlwithHTTPRefreshMeta = 'http://www.windowsphone.com/en-US/apps/82a23635-5bd9-df11-a844-00237de2db9e';

suite.addBatch({
   'Load a scrapule in scrapinode' : {
      topic : function(){
         var useScrapules = function(){ scrapinode.use(pathScrapulesDir)}
         return useScrapules
      },
      'should happen without throwing errors' : function(useScrapules){
         assert.doesNotThrow(useScrapules,Error)
      }
   }
}).addBatch({
   'Create a scraper' : {
      'to scrap an HTML page thanks to its url' : {
         topic : function(){
            scrapinode.init();
            scrapinode.createScraper(urlDefault,this.callback);
         },
         'should happen without error' : function(err,scraper){
            assert.isNull(err);
         },
         'should give a Scraper object': function(err,scraper){
            assert.instanceOf(scraper,Scraper);
         },
         'should give a Scraper object able to retrieve the title of the page' : function(err,scraper){
            assert.isString(scraper.get('title'));
         },
         'should give a Scraper object able to retrieve a description of the page' : function(err,scraper){
            assert.isArray(scraper.get('description'));
         },
         'should give a Scraper object able to retrieve images of the page' : function(err,scraper){
            assert.isArray(scraper.get('images'));
         }
      },
      'to scrap an HTML page thanks to its url using a specific scrapule determined by the domain name of the page' : {
         topic : function(){
            scrapinode.use(pathScrapulesDir);
            scrapinode.createScraper(urlCultBeauty,this.callback)
         },
         'should happen without error' : function(err,scraper){
            assert.isNull(err);
         },
         'should give a Scraper object' : function(err,scraper){
            assert.instanceOf(scraper,Scraper);
         },
         'should give a Scraper object able to retrieve the title of the page' : function(err,scraper){
            assert.isString(scraper.get('title'));
         },
         'should give a Scraper object able to retrieve a description of the page' : function(err,scraper){
            assert.isArray(scraper.get('description'));
         },
         'should give a Scraper object able to retrieve images of the page' : function(err,scraper){
            assert.isArray(scraper.get('images'));
         },
         'should give a Scraper object able to retrieve the price of the product' : function(err,scraper){
            assert.match(scraper.get('price'),/([0-9,]{1,}(\.?[0-9]{1,}))/)
         }
      },
      'to scrap an HTML page created because the resource found with the link given is an image' : {
         topic : function(){
            scrapinode.createScraper(urlImage,this.callback)
         },
         'should happen without error' : function(err,scraper){
            assert.isNull(err);
         },
         'should give a Scraper object' : function(err,scraper){
            assert.instanceOf(scraper,Scraper);
         },
         'should give a Scraper object able to retrieve images of the page' : function(err,scraper){
            assert.isArray(scraper.get('images'));
         },
      },
      'to scrap an HTML page with in it a HTTP-REFRESH meta tag' : {
         topic : function(){
            scrapinode.createScraper(urlwithHTTPRefreshMeta,this.callback)
         },
         'should give a Scraper object' : function(err,scraper){
            assert.instanceOf(scraper,Scraper);
         },
         'should give a Scraper object able to retrieve the title of the page' : function(err,scraper){
            assert.isString(scraper.get('title'));
         },
         'should give a Scraper object able to retrieve a description of the page' : function(err,scraper){
            assert.isArray(scraper.get('description'));
         },
         'should give a Scraper object able to retrieve images of the page' : function(err,scraper){
            assert.isArray(scraper.get('images'));
         }
      }
   }
   
}).export(module)

{'fraction_non_alphanumeric': 0.12154907975460123, 'fraction_numerical': 0.008052147239263804, 'mean_word_length': 2.4808718861209966, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 5, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6476651', 'n_tokens_mistral': 2344, 'n_tokens_neox': 2361, 'n_words': 574}