<commit_before>var phantom = require('phantom')
  , http = require('http');

phantom.create(function(phantom) {
    http.createServer(function (req, res) {
        console.log('getting', req.url);
        phantom.createPage(function(page) {
            page.open(req.url.substr(1), function (status) {
                if ('fail' === status) { 
                    res.writeHead(404);
                    res.end();
                    page.close();
                } else {
                    setTimeout(function(){
                        page.evaluate(function () {
                            return document && document.getElementsByTagName('html')[0].outerHTML
                        }, function(documentHTML) {
                            var matches = documentHTML.match(/<script(?:.*?)>(?:[\S\s]*?)<\/script>/g);

                            for( var i = 0; matches && i < matches.length; i++) {
                                documentHTML = documentHTML.replace(matches[i], '');
                            }
                            res.writeHead(200, {'Content-Type': 'text/html'});
                            res.end(documentHTML);
                            page.close();
                        });
                    }, 50);
                };
            });
        });
    }).listen(process.env.PORT || 3000);
    console.log('Server running on port ' + (process.env.PORT || 3000));
});<commit_msg>workaround for possibility of a phantomjs crash...for now<commit_after>var Phantom = require('phantom')
  , http = require('http');

Phantom.create(function(phantom) {

    http.createServer(function (req, res) {
        console.log('getting', req.url);

        //hack to restart phantom if it crashes...for now
        var timeoutID = setTimeout(function() {
            console.log('restarting phantom due to timeout');
            Phantom.create(function(ph){
                phantom = ph;
                phantom.createPage(onPageCreate);
            });
        }, 5000);

        var onPageCreate = function(page) {
            clearTimeout(timeoutID); //wont get hit if phantom has crashed

            page.open(req.url.substr(1), function (status) {
                if ('fail' === status) { 
                    res.writeHead(404);
                    res.end();
                    page.close();
                } else {
                    setTimeout(function(){
                        page.evaluate(function () {
                            return document && document.getElementsByTagName('html')[0].outerHTML
                        }, function(documentHTML) {
                            var matches = documentHTML.match(/<script(?:.*?)>(?:[\S\s]*?)<\/script>/g);

                            for( var i = 0; matches && i < matches.length; i++) {
                                documentHTML = documentHTML.replace(matches[i], '');
                            }
                            res.writeHead(200, {'Content-Type': 'text/html'});
                            res.end(documentHTML);
                            page.close();
                        });
                    }, 50);
                };
            });
        };

        phantom.createPage(onPageCreate);

    }).listen(process.env.PORT || 3000);
    console.log('Server running on port ' + (process.env.PORT || 3000));
});
{'fraction_non_alphanumeric': 0.13956274333632823, 'fraction_numerical': 0.012578616352201259, 'mean_word_length': 1.3050379572118702, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1604454', 'n_tokens_mistral': 890, 'n_tokens_neox': 791, 'n_words': 149}