<commit_before>/**
 * TODO exceptions should exit immediately from phantomjs and they DON'T
 * TODO add a timeout for too long connections
 * TODO support 404 and error code in the same way prerender.io (server) does
 * @type {number}
 */
var EVALUATE_JAVASCRIPT_CHECK_TIMEOUT = 50;

var page = require('webpage').create(),
    system = require('system');
var surrender = {};
page.viewportSize = {
  width: 1024,
  height: 800
};

surrender.params = function() {
    if (system.args.length === 1) {
        throw 'Usage: googlizer.js <some URL>';
    }
    return { address: system.args[1]};
};

surrender.javascriptToExecuteOnPage = function() {
    var fillHTMLRecursive = function(parent, bool) {
        var node, x, _results;
        var _that = this;

        // bb = document.getElementsByTagName('body')[0]


        // if(bool){
        //     bb.innerHTML = 'true'
        // }
        // _results = [];
        x = 0;
        while (x < parent.childNodes.length) {
            node = parent.childNodes[x];

            debugger
            if (node.childNodes.length === 0) {
                x++;
                continue;
            }
            else {
                elem = node.innerText;
                if (elem != null && elem.length > 1) {
                    console.log('else ', elem);
                    // fillHTMLRecursive(node, true);
                };
                
            }        
            x++;
        }
        // return _results;
    };
    try {
        
        var doctype = '';
        var html = '';
            // , html = document && document.getElementsByTagName('html');
        if(document.getElementById('articles-show') != null) {
            // articles
            
            article = document.getElementsByTagName('article')[0];
            // articleParent = article.parentElement;
            text = article.getElementsByClassName('text')[0];
            textLength = text.innerText.length;

            // nodes = text.childNodes;
            // nodesNumber = nodes.length;
            // count = 0;

            // currentLength = 0;

            fillHTMLRecursive(text, false);

            // while ( count < nodesNumber) {
            //     elem = nodes[count];
            //     console.log(count, ' ', elem);
            //     if (elem.innerText) {
            //         elem.innerText
            //     };
            //     count++;
            // }

            // metto il 30% del testo
            // article.innerHTML = text.innerText.substring(0, Math.round(textLength * 30 / 100)) + '...';
            // calcolo la lunghezza del testo ridotto al 30%
            // maxLen = article.innerHTML.length;
            // article.innerHTML = maxLen;

            html = document && document.getElementsByTagName('head');
<commit_msg>remove elem after limit<commit_after>/**
 * TODO exceptions should exit immediately from phantomjs and they DON'T
 * TODO add a timeout for too long connections
 * TODO support 404 and error code in the same way prerender.io (server) does
 * @type {number}
 */
var EVALUATE_JAVASCRIPT_CHECK_TIMEOUT = 50;

var page = require('webpage').create(),
    system = require('system');
var surrender = {};
page.viewportSize = {
  width: 1024,
  height: 800
};

surrender.params = function() {
    if (system.args.length === 1) {
        throw 'Usage: googlizer.js <some URL>';
    }
    return { address: system.args[1]};
};

surrender.javascriptToExecuteOnPage = function() {
    var fillHTMLRecursive = function(parent, limit) {
        var node, x, _results;
        var _that = this;
        var countChar = 0;
        bb = document.getElementsByTagName('body')[0]
        x = 0;
        while (x < parent.childNodes.length) {
            debugger
            node = parent.childNodes[x];
            elem = node.data;
            if(elem === undefined) {
                elem = node.innerText;
            }

            // Se il limite (70%) e' superato, elimino tt i nodi seguenti
            if (countChar > limit) {
                parent.removeChild(node);
                x++;
                continue;
            };

            if (elem != null && elem.length > 1) {
                countChar = countChar + elem.length;
                if (countChar > limit) {
                    parent.removeChild(node);
                };
                // fillHTMLRecursive(node, true);
            };




            // debugger
            // if (node.childNodes.length === 0) {
            //     x++;
            //     continue;
            // }
            // else {

            //     // elem = node.data;

            //     if (elem != null && elem.length > 1) {
            //         countChar = countChar + elem.length;
            //         if (countChar > limit) {
            //             parent.removeChild(node);
            //         };

            //         // fillHTMLRecursive(node, true);
            //     };
                
            // }        
            x++;
        }
        // return _results;
    };


    try {
        
        var doctype = '';
        var html = '';
            // , html = document && document.getElementsByTagName('html');
        if(document.getElementById('articles-show') != null) {
            // articles
            
            article = document.getElementsByTagName('article')[0];
            // articleParent = article.parentElement;
            text = article.getElementsByClassName('text')[0];
            textLength = text.innerText.length;
            limit = Math.round(textLength * 70 / 100);

            // text.innerText.substring(0, Math.round(textLength * 70 / 100)) + '...';

            // nodes = text.childNodes;
            // nodesNumber = nodes.length;
            // count = 0;

            // currentLength = 0;

            fillHTMLRecursive(text, limit);

            // while ( count < nodesNumber) {
            //     elem = nodes[count];
            //     console.log(count, ' ', elem);
            //     if (elem.innerText) {
            //         elem.innerText
            //     };
            //     count++;
            // }

            // metto il 30% del testo
            // article.innerHTML = text.innerText.substring(0, Math.round(textLength * 30 / 100)) + '...';
            // calcolo la lunghezza del testo ridotto al 30%
            // maxLen = article.innerHTML.length;
            // article.innerHTML = maxLen;

            html = document && document.getElementsByTagName('head');

{'fraction_non_alphanumeric': 0.1273983115886416, 'fraction_numerical': 0.01212586339217191, 'mean_word_length': 1.3807088052612349, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 11, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '644646', 'n_tokens_mistral': 1844, 'n_tokens_neox': 1717, 'n_words': 401}