<commit_before>var request = require('request'),
	j2h = require('json2html'),
	fs = require('fs'),
	w3url = 'http://localhost/w3c-validator/check',
	uriOpt = '?uri=',
	outputOpt = '&output=json',
	saveFolder = 'checked/',
	changedFolder = 'changed/',
	diffsFound = 0,
	errors = {},
	urlsEnv = process.env.URLS,
	buildNo = process.env.BUILD_NUMBER,
	urls = [];

var NEW_ERROR = "new",
	OLD_ERROR = "gone";

if( urlsEnv && urlsEnv.length > 0 ) {
	urls = urlsEnv.split(' ');
	processUrls();
} else {
	console.log( "Please provide a list of URLs as space-separated list in the environment variable URLS." );
	process.exit(1);
}

process.on('exit', function() {
	console.log( "Done." );
	process.exit( diffsFound );
});

function processUrls( ) {
	urls.forEach( function( url ) {
		console.log( "Checking " + url );
		errors[ url ] = { 
			nu: false,
			ol: false,
			changes: []
		};
		readOldErrors( url );
		getW3sOpinion( url );
	});
}

function readOldErrors( url ) {
	var fileName = getJsonFileName( url );
	fs.exists( fileName, function( exists ) {
		if( !exists ) {
			console.log( "Old file doesn't exist" );
			errors[ url ].ol = {};
		} else {
			console.log( "Old file exists");
			fs.readFile( fileName, function( err, data ) {
				if( err ) console.log( "Unable to read file " + fileName );
				else {
					if( data && data.length ) {
						try {
							var j = JSON.parse( data );
							console.log( "Setting old data for " + url );
							errors[ url ].ol = j;
						} catch( e ) {
							console.log( e );
						}
					} else {
						console.log( " - but it's empty!");
						errors[ url ].ol = {};
					}
				}
			});
		}
		compareAndSaveIfDone();
	});
}

function getW3sOpinion( url ) {
	request( w3url + uriOpt + url + outputOpt, function( error, resp, body ) {
		if( !error ) {
			if( resp && resp.statusCode == 200 ) {
				try {
					var j = JSON.parse( body );
					console.log( "Setting new data for " + url );
					errors[ url ].nu = j;
				} catch( e ) {
					console.log( "Error in parsing new data " + e );
					console.log( "Data: " + body );
					errors[ url ].nu = {};
				}
			}
		} else {
			console.log( "Error in request for " + url + ": " + error );
			errors[ url ].nu = {};
		}
		compareAndSaveIfDone();
	});
}

function compareAndSaveIfDone() {
	if( areWeDone() ) {
		Object.keys( errors ).forEach( function( url ) {
			console.log( "Checking for data of " + url );
			if( findChanges( errors[url].nu, errors[url].ol, url ) ) {
				console.log( "Saving for " + url );
				saveFile( url );
				diffsFound = 1;
			} else {
				console.log( "No differences found for " + url );
			}
		});
	}
}

function findChanges( nu, ol, url ) {
	var changesFound = false;
	console.log( "Comparing " );
	if( !nu || !ol ) {
		console.log( "Something went wrong. Very wrong. nu: " + nu + ", ol: " + ol);
		return true;
	}

	if( nu.messages ) {
		if( nu.messages.length && ( ! ol.messages || ! ol.messages.length ) ) {
			console.log( "No old errors - all new!" );
			return true;
		}

		if( ol.messages && ol.messages.length ) {
			if( nu.messages.length >= ol.messages.length ) {
				nu.messages.forEach( function( n ) {
					if( ! ol.messages.some( function( o ) {
						if( o.message == n.message && o.lastLine == n.lastLine ) {
							return true;
						}
					}) && isNotJustInfo( n ) ) {
						n.changeType = NEW_ERROR;
						errors[ url ].changes.push(n);
						changesFound = true;
					}
				});
				// less errors
			} else {
				ol.messages.forEach( function( o ) {
					if( ! nu.messages.some( function( n ) {
						if( n.message == o.message && n.lastLine == o.lastLine ) {
							return true;<commit_msg>Change handling when no change in number of errors<commit_after>var request = require('request'),
	j2h = require('json2html'),
	fs = require('fs'),
	w3url = 'http://localhost/w3c-validator/check',
	uriOpt = '?uri=',
	outputOpt = '&output=json',
	saveFolder = 'checked/',
	changedFolder = 'changed/',
	diffsFound = 0,
	errors = {},
	urlsEnv = process.env.URLS,
	buildNo = process.env.BUILD_NUMBER,
	urls = [];

var NEW_ERROR = "new",
	OLD_ERROR = "gone";

if( urlsEnv && urlsEnv.length > 0 ) {
	urls = urlsEnv.split(' ');
	processUrls();
} else {
	console.log( "Please provide a list of URLs as space-separated list in the environment variable URLS." );
	process.exit(1);
}

process.on('exit', function() {
	console.log( "Done." );
	process.exit( diffsFound );
});

function processUrls( ) {
	urls.forEach( function( url ) {
		console.log( "Checking " + url );
		errors[ url ] = { 
			nu: false,
			ol: false,
			changes: []
		};
		readOldErrors( url );
		getW3sOpinion( url );
	});
}

function readOldErrors( url ) {
	var fileName = getJsonFileName( url );
	fs.exists( fileName, function( exists ) {
		if( !exists ) {
			console.log( "Old file doesn't exist" );
			errors[ url ].ol = {};
		} else {
			console.log( "Old file exists");
			fs.readFile( fileName, function( err, data ) {
				if( err ) console.log( "Unable to read file " + fileName );
				else {
					if( data && data.length ) {
						try {
							var j = JSON.parse( data );
							console.log( "Setting old data for " + url );
							errors[ url ].ol = j;
						} catch( e ) {
							console.log( e );
						}
					} else {
						console.log( " - but it's empty!");
						errors[ url ].ol = {};
					}
				}
			});
		}
		compareAndSaveIfDone();
	});
}

function getW3sOpinion( url ) {
	request( w3url + uriOpt + url + outputOpt, function( error, resp, body ) {
		if( !error ) {
			if( resp && resp.statusCode == 200 ) {
				try {
					var j = JSON.parse( body );
					console.log( "Setting new data for " + url );
					errors[ url ].nu = j;
				} catch( e ) {
					console.log( "Error in parsing new data " + e );
					console.log( "Data: " + body );
					errors[ url ].nu = {};
				}
			}
		} else {
			console.log( "Error in request for " + url + ": " + error );
			errors[ url ].nu = {};
		}
		compareAndSaveIfDone();
	});
}

function compareAndSaveIfDone() {
	if( areWeDone() ) {
		Object.keys( errors ).forEach( function( url ) {
			console.log( "Checking for data of " + url );
			if( findChanges( errors[url].nu, errors[url].ol, url ) ) {
				console.log( "Saving for " + url );
				saveFile( url );
				diffsFound = 1;
			} else {
				console.log( "No differences found for " + url );
			}
		});
	}
}

function findChanges( nu, ol, url ) {
	var changesFound = false;
	console.log( "Comparing " );
	if( !nu || !ol ) {
		console.log( "Something went wrong. Very wrong. nu: " + nu + ", ol: " + ol);
		return true;
	}

	if( nu.messages ) {
		if( nu.messages.length && ( ! ol.messages || ! ol.messages.length ) ) {
			console.log( "No old errors - all new!" );
			return true;
		}

		if( ol.messages && ol.messages.length ) {
			if( nu.messages.length > ol.messages.length ) {
				nu.messages.forEach( function( n ) {
					if( ! ol.messages.some( function( o ) {
						if( o.message == n.message && o.lastLine == n.lastLine ) {
							return true;
						}
					}) && isNotJustInfo( n ) ) {
						n.changeType = NEW_ERROR;
						errors[ url ].changes.push(n);
						changesFound = true;
					}
				});
				// less errors
			} else if( nu.messages.length == ol.messages.length ) {
				// check for different error message
				nu.messages.forEach( function( n ){
					if( ! ol.messages.some( function( o ) {
						if( o.message == n.message ) {
							return true;
						}
					}) && isNotJustInfo( n ) ) {
						n.changeType = NEW_ERROR;
						errors[ url ].changes.push(n);
						changesFound = true;
					}
				});
			}
			else {
				ol.messages.forEach( function( o ) {
					if( ! nu.messages.some( function( n ) {
						if( n.message == o.message && n.lastLine == o.lastLine ) {
							return true;
{'fraction_non_alphanumeric': 0.16952232606438214, 'fraction_numerical': 0.0036344755970924196, 'mean_word_length': 2.4990917347865578, 'pattern_counts': {'":': 2, '<': 3, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2039862', 'n_tokens_mistral': 3269, 'n_tokens_neox': 2707, 'n_words': 697}