<commit_before>// END of ParseUri
// BEGIN modified netlog.js example by PhantomJS
// Source: https://github.com/ariya/phantomjs/blob/master/examples/netlog.js
// Source of PhantomJS: https://github.com/ariya/phantomjs
// Original Code distributed under the BSD License

var page = require('webpage').create(),
    system = require('system'),
    address, set = {};
try {
	
	if (system.args.length === 1) {
	    console.log('Usage: netdomain.js <some URL>');
	    phantom.exit(1);
	} else {
	    address = system.args[1];
	
	    page.onResourceReceived = function (res) {
	        set[JSON.stringify(res.url, undefined, 4).split("/")[2]] = true;
	    };
	    page.onError = function(err) {
	    };
	
	    page.open(address, function (status) {
	        if (status !== 'success') {
	            console.log('FAIL to load the address');
		    phantom.exit(1);
	        }
		console.log(Object.keys(set).toString());
	        phantom.exit();
	    });
	}
	// END Modified netlog.js example
}
catch(err) {
	console.log(Object.keys(set).toString());
	phantom.exit();
}
<commit_msg>Modified the script with some SICK JS for link validation :P<commit_after>// END of ParseUri
// BEGIN modified netlog.js example by PhantomJS
// Source: https://github.com/ariya/phantomjs/blob/master/examples/netlog.js
// Source of PhantomJS: https://github.com/ariya/phantomjs
// Original Code distributed under the BSD License

var page = require('webpage').create(),
    system = require('system'),
    address, set = {};
try {
	
	if (system.args.length === 1) {
	    console.log('Usage: netdomain.js <some URL>');
	    phantom.exit(1);
	} else {
	    address = system.args[1];
	
	    page.onResourceReceived = function (res) {
	        var data = JSON.stringify(res.url, undefined, 4).split("/")[2];
		if (~data.indexOf(".")) {
		    set[data] = true;
		}
	    };
	    page.onError = function(err) {
	    };
	
	    page.open(address, function (status) {
	        if (status !== 'success') {
	            console.log('FAIL to load the address');
		    phantom.exit(1);
	        }
		console.log(Object.keys(set).toString());
	        phantom.exit();
	    });
	}
	// END Modified netlog.js example
}
catch(err) {
	console.log(Object.keys(set).toString());
	phantom.exit();
}

{'fraction_non_alphanumeric': 0.16622222222222222, 'fraction_numerical': 0.005333333333333333, 'mean_word_length': 2.8217317487266556, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 4, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '629412', 'n_tokens_mistral': 781, 'n_tokens_neox': 704, 'n_words': 159}