<commit_before><commit_msg>added scrapper<commit_after>/**
 * Created by prism on 4/15/16.
 */
var request = require('request'),
    cheerio = require('cheerio');


module.exports = function () {
    var url = 'http://joblist.bdjobs.com/jobsearch.asp?fcatId=8';
    var j = request.jar();
    j.setCookie(request.cookie('JOBSRPP=40'), url);   //set 40 results per page

    var options = {
        url: url,
        headers: {
            'User-Agent': 'Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:48.0) Gecko/20100101 Firefox/48.0'
        },
        jar: j
    };

    request(options, function (error, response, body) {
        if (!error && response.statusCode == 200) {
            $ = cheerio.load(body);
            var res = $('.job_title_text').text();
            console.log(res);
            console.log(res.length);
        }
    })
}
{'fraction_non_alphanumeric': 0.16312056737588654, 'fraction_numerical': 0.041371158392434985, 'mean_word_length': 2.2452107279693485, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2440214', 'n_tokens_mistral': 307, 'n_tokens_neox': 277, 'n_words': 58}