<commit_before>"use strict";

let fs = require('fs');
let Horseman = require('node-horseman');
let jsdom = require("jsdom");
let ical = require('ical-generator');

let config = require('./config');

// Main entry point
// Load all pages from the website
function loadPages(parsingCallback, callback) {
	console.log('icalCreator:loadPages : loading all pages');
	let pages = getPagesList();
	//console.log('icalCreator:loadPages : page list: ' + JSON.stringify(pages));
	getPagesList(pages, parsingCallback, {}, callback);
}

// Requesthandler: load each url in a synchronous way
function requestHandler(urls, parsingCallback, htmlMap, callback) {
	console.log('icalCreator:requestHandler : remaining length: ' + Object.keys(urls).length);
	if (Object.keys(urls).length == 0) {
		parsingCallback(htmlMap, callback);
	} else {
		let firstKey = Object.keys(urls)[0];
		requestPage(firstKey, urls[firstKey], firstKey, function(html) {<commit_msg>- fix bug (call requestHandler)<commit_after>"use strict";

let fs = require('fs');
let Horseman = require('node-horseman');
let jsdom = require("jsdom");
let ical = require('ical-generator');

let config = require('./config');

// Main entry point
// Load all pages from the website
function loadPages(parsingCallback, callback) {
	console.log('icalCreator:loadPages : loading all pages');
	let pages = getPagesList();
	//console.log('icalCreator:loadPages : page list: ' + JSON.stringify(pages));
	requestHandler(pages, parsingCallback, {}, callback);
}

// Requesthandler: load each url in a synchronous way
function requestHandler(urls, parsingCallback, htmlMap, callback) {
	console.log('icalCreator:requestHandler : remaining length: ' + Object.keys(urls).length);
	if (Object.keys(urls).length == 0) {
		parsingCallback(htmlMap, callback);
	} else {
		let firstKey = Object.keys(urls)[0];
		requestPage(firstKey, urls[firstKey], firstKey, function(html) {
{'fraction_non_alphanumeric': 0.14653361344537816, 'fraction_numerical': 0.0021008403361344537, 'mean_word_length': 5.85251798561151, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2326581', 'n_tokens_mistral': 583, 'n_tokens_neox': 566, 'n_words': 150}