<commit_before>var config  = require('./config'),
    log     = require('./log');

/*
 TODO:
  - Add some example sites.
 */

rippers = Array();

require('fs').readdirSync(__dirname + '/rippers/').forEach(function(file) {
	log.debug('[Ripper Autoload] Found file: ' + file);

	if (file.match(/^.+\.js$/) !== null && file !== 'index.js') {
		ripper = require('./rippers/' + file);
		log.debug('[Ripper Autoload] Adding ripper: ' + ripper.name);
		rippers.push(ripper);
	}
});

config.sites.forEach(function (site) {
	if (site.action != 'disabled') {
		var ripper = null, authority = -1;
		rippers.forEach(function (item) {
			if (item.url.test(site.url)) {
				log.debug(site.name + ' - Found matching ripper: ' + item.name);
				if (item.authority > authority) {
					ripper = item;
					authority = item.authority;
					log.debug(site.name + ' - Choosing ripper: ' + item.name + ' ...');
				}
			}
		});

		if (ripper !== null) {
			log.info(site.name + ' - Starting scrape with ripper ' + ripper.name + '...');
			ripper.rip(site);
		} else {
			log.error(site.name + ' - Could not find valid ripper. Skipping...');
		}
	} else {
		log.info(site.name + ' - Site is disabled. Skipping...')
	}
});<commit_msg>Improve readability of app.js ripper log messages.<commit_after>var config  = require('./config'),
    log     = require('./log');

/*
 TODO:
  - Add some example sites.
 */

rippers = Array();

require('fs').readdirSync(__dirname + '/rippers/').forEach(function(file) {
	log.debug('[Ripper Autoload] Found file: ' + file);

	if (file.match(/^.+\.js$/) !== null && file !== 'index.js') {
		ripper = require('./rippers/' + file);
		log.debug('[Ripper Autoload] Adding ripper: ' + ripper.name);
		rippers.push(ripper);
	}
});

config.sites.forEach(function (site) {
	if (site.action != 'disabled') {
		var ripper = null, authority = -1;
		rippers.forEach(function (item) {
			if (item.url.test(site.url)) {
				log.debug(site.name + ' - Found matching ripper: ' + item.name);
				if (item.authority > authority) {
					ripper = item;
					authority = item.authority;
					log.debug(site.name + ' - Choosing ripper: ' + item.name);
				}
			}
		});

		if (ripper !== null) {
			log.info(site.name + ' - Starting scrape with ripper: ' + ripper.name);
			ripper.rip(site);
		} else {
			log.error(site.name + ' - Could not find valid ripper. Skipping.');
		}
	} else {
		log.info(site.name + ' - Site is disabled. Skipping.')
	}
});
{'fraction_non_alphanumeric': 0.1990151826015593, 'fraction_numerical': 0.0008206811653672548, 'mean_word_length': 2.957792207792208, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4339008', 'n_tokens_mistral': 967, 'n_tokens_neox': 862, 'n_words': 180}