<commit_before>var page = new WebPage();
var system = require('system');
var url = 'http://twitter.com/search/javascript';
var timeout = 8000;  

function displayHelp () {
    console.log('Usage:');
    console.log(system.args[0] + ' \'http://twitter.com/search/javascript\'');
    phantom.exit();
}


function argParser () {
    if (system.args.length === 1) {        
        displayHelp();
        phantom.exit();          
    } else {
        //console.log(system.args.length);
        url = system.args[1];
        if (system.args[2] != 'undefined') {
            timeout = system.args[2];
        }
        //phantom.exit();        
    }
}

argParser();

page.open(url, function (status) {
    
    
    
    if (status == 'success') {
        // inject jquery
        //page.injectJs("http://code.jquery.com/jquery-latest.min.js", function() {
    // jQuery is loaded, now manipulate the DOM
        //});
        getFullDom();
    } else {
       console.log('failure open page');
       phantom.exit();
    }
});




function getFullDom() {
    window.setTimeout(function () {
        var results = page.evaluate(function() {
            return document.getElementsByTagName('html')[0].innerHTML;
            // If jquery is loaded you can use this: 
            //return $('html').html();
        });
        console.log(results);
        phantom.exit();
    }, timeout);
}<commit_msg>try and catch and phantomjs grab-dom.js<commit_after>var page = new WebPage();
var system = require('system');
var url = 'http://twitter.com/search/javascript';
var timeout = 8000;  

function displayHelp () {
    console.log('Usage:');
    console.log(system.args[0] + ' \'http://twitter.com/search/javascript\'');
    phantom.exit();
}

function argParser () {
    if (system.args.length === 1) {        
        displayHelp();
        phantom.exit();          
    } else {
        //console.log(system.args.length);
        url = system.args[1];
        if (system.args[2] != 'undefined') {
            timeout = system.args[2];
        }       
    }
}

argParser();

try {
    page.open(url, function (status) {
        
        if (status == 'success') {
            // inject jquery
            //page.injectJs("http://code.jquery.com/jquery-latest.min.js", function() {
            // jQuery is loaded, now manipulate the DOM
            //});
            getFullDom();
        } else {
            console.log('failure open page');
            phantom.exit();
        }
    });

} catch (err) {
    console.log(err.message);
    phantom.exit();
}

function getFullDom() {
    window.setTimeout(function () {
        var results = page.evaluate(function() {
            return document.getElementsByTagName('html')[0].innerHTML;
            // If jquery is loaded you can use this: 
            //return $('html').html();
        });
        console.log(results);
        phantom.exit();
    }, timeout);
}
{'fraction_non_alphanumeric': 0.16449175824175824, 'fraction_numerical': 0.006868131868131868, 'mean_word_length': 1.936491935483871, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1259757', 'n_tokens_mistral': 886, 'n_tokens_neox': 835, 'n_words': 171}