<commit_before>"use strict";
const request = require('request-promise');
const cheerio = require('cheerio');

module.exports = function () {
    let promises = [];
    for(let offset = 0 ; offset < 1000 ; offset += 50){
        promises.push(request({
            uri: 'http://proxydb.net/',
            qs: {
                limit: 50,
                offset
            }
        }).then((body) => {
            cheerio.load(body);
        }).then(($) => {
            if(!$)
                return [];
            return $("table > tbody > tr").map((i , element) => {
                let ip = $(element).find('td > a').first().text().trim();
                let port = $(element).find('td').first().next().text();
                console.log("ip " + ip + " port " + port);
                return { ip , port }
            }).get();
        }));
    }
    return Promise.all(promises).then((values) => values.reduce((prev , next) => prev.concat(next)) , []);
};<commit_msg>Update proxydb.net.js<commit_after>"use strict";
const request = require('request-promise');
const cheerio = require('cheerio');

module.exports = function () {
    let promises = [];
    for(let offset = 0 ; offset < 1000 ; offset += 50){
        promises.push(request({
            uri: 'http://proxydb.net/',
            qs: {
                limit: 50,
                offset: 0
            }
        })
        .then((body) => cheerio.load(body))
        .then(($) => {
            if(!$)
                return [];
            return $("table > tbody > tr").map((i , element) => {
                let ip = $(element).find('td > a').first().text().trim();
                let port = $(element).find('td').first().next().text().trim();
                return { ip , port }
            }).get();
        }));
    }
    return Promise.all(promises).then((values) => values.reduce((prev , next) => prev.concat(next)) , []);
};
{'fraction_non_alphanumeric': 0.19663512092534174, 'fraction_numerical': 0.009989484752891693, 'mean_word_length': 1.6690042075736324, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 19, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '939903', 'n_tokens_mistral': 564, 'n_tokens_neox': 534, 'n_words': 110}