<commit_before>var http = require('http'),
    url_mod = require('url'),
    fs = require('fs');
        
var readability = require('../lib/readability.js'),
    sprintf = readability.sprintf;

function cleanFile(path, url, cb) {
    var content = fs.readFileSync(path, 'utf-8');
    readability.parse(content, url, cb);
}

// cleanFile(__dirname + '/pages/http___www_youtube_com_watch_v_HBnNoF71W2Q.html', '', function(info) {
//     //console.log(info.content);
// });
// 
// process.exit();

function batch_run() {
    var dir = __dirname + '/pages/';
    var files = fs.readdirSync(dir);
    var results = [];
    //files.length = 10;
    files.forEach(function(f) {
        console.log('######## Processing file...', f);
        cleanFile(dir + f, '', function(result) {
            results.push({time: result.time, file: f, inputLength: result.inputLength, error: result.error});
        });
    });
    
    var total = 0, totalTime = 0;
    results.filter(function(v) {return !v.error}).sort(function(a, b) {return a.time - b.time;}).forEach(function(r) {
        total++;
        totalTime += r.time;
        console.log(sprintf('%5.2f\t%8d\t%10s', r.time, r.inputLength, r.file));
    });
    console.log('total:', total, "avg time:", totalTime/total);
}

batch_run();<commit_msg>process.exit() exits immediately. return is nicer<commit_after>var http = require('http'),
    url_mod = require('url'),
    fs = require('fs');
        
var readability = require('../lib/readability.js'),
    sprintf = readability.sprintf;

function cleanFile(path, url, cb) {
    var content = fs.readFileSync(path, 'utf-8');
    readability.parse(content, url, cb);
}

// cleanFile(__dirname + '/pages/Proyectos_de_creaci_n_de_empresas___til_para_el_emprendedor__Ofertas____.html', '', function(info) {
//     //console.log(info.content);
// });
// 
// return;

function batch_run() {
    var dir = __dirname + '/pages/';
    var files = fs.readdirSync(dir);
    var results = [];
    //files.length = 10;
    files.forEach(function(f) {
        console.log('######## Processing file...', f);
        cleanFile(dir + f, '', function(result) {
            results.push({time: result.time, file: f, inputLength: result.inputLength, error: result.error});
        });
    });
    
    var total = 0, totalTime = 0;
    results.filter(function(v) {return !v.error}).sort(function(a, b) {return a.time - b.time;}).forEach(function(r) {
        total++;
        totalTime += r.time;
        console.log(sprintf('%5.2f\t%8d\t%10s', r.time, r.inputLength, r.file));
    });
    console.log('total:', total, "avg time:", totalTime/total);
}

batch_run();
{'fraction_non_alphanumeric': 0.1899166034874905, 'fraction_numerical': 0.008718726307808946, 'mean_word_length': 3.078825347758887, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 1, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4849263', 'n_tokens_mistral': 898, 'n_tokens_neox': 886, 'n_words': 168}