<commit_before>/**
 * Created by Magador on 18/04/2015.
 */

var downloader = require('downloader'),
    htmlparser = require('htmlparser2');

var requestPool = {
    requests: [],
    currents: [],
    size: 5,
    add: function(url) {
        if(url instanceof Array) {
            url.forEach(function (val) {
                requestPool.add(val);
            });<commit_msg>Remove methd requestPool<commit_after>/**
 * Created by Magador on 18/04/2015.
 */

var downloader = require('downloader'),
    htmlparser = require('htmlparser2');

var requestPool = {
    requests: [],
    currents: [],
    size: 5,
    add: function(url) {
        if(Array.isArray(url)) {
            url.forEach(function (val) {
                requestPool.add(val);
            });
        }
        if(typeof url === 'string') {
            requestPool.requests.push({
                url: url,
                done: false
            });
        }
    },
    remove: function(url) {
        if(Array.isArray(url)) {
            var removingUrls = url.filter(function(val) {
                return true;
            });
{'fraction_non_alphanumeric': 0.1403985507246377, 'fraction_numerical': 0.018115942028985508, 'mean_word_length': 1.7694235588972431, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5535859', 'n_tokens_mistral': 335, 'n_tokens_neox': 317, 'n_words': 66}