<commit_before>var phantom = require('phantom')
  , http = require('http')
  , url = require('url')
  , _ = require('lodash');

var DEFAULT_PHANTOM_ARGUMENTS = ['--load-images=false', '--ignore-ssl-errors=true'];

var PAGE_DONE_CHECK_TIMEOUT = process.env.PAGE_DONE_CHECK_TIMEOUT || 50;

var RESOURCE_DOWNLOAD_TIMEOUT = process.env.RESOURCE_DOWNLOAD_TIMEOUT || 10 * 1000;

var WAIT_AFTER_LAST_REQUEST = process.env.WAIT_AFTER_LAST_REQUEST || 500;

var JS_CHECK_TIMEOUT = process.env.JS_CHECK_TIMEOUT || 50;

var JS_TIMEOUT = process.env.JS_TIMEOUT || 10 * 1000;

var EVALUATE_JAVASCRIPT_CHECK_TIMEOUT = process.env.EVALUATE_JAVASCRIPT_CHECK_TIMEOUT || 50;

// Normalizes unimportant differences in URLs - e.g. ensures
// http://google.com/ and http://google.com normalize to the same string
function normalizeUrl(u) {
    return url.format(url.parse(u, true));
}

var server = exports = module.exports = {};

server.init = function(options) {
    this.plugins = this.plugins || [];
    this.options = options;

    return this;
};

server.start = function() {
    if(!this.options.isMaster) {
        this.createPhantom();
    }
};

server.use = function(plugin) {
    this.plugins.push(plugin);
    if (typeof plugin.init === 'function') plugin.init(this);
};

server._pluginEvent = function(methodName, args, callback) {
    var _this = this
      , index = 0
      , next;

    next = function() {
        var layer = _this.plugins[index++];
        if (!layer) return callback();

        var method = layer[methodName];

        if (method) {
            method.apply(layer, args);
        } else {
            next();
        }
    }
    
    args.push(next);
    next();
};

server.createPhantom = function() {
    var _this = this;
    console.log('starting phantom')

    phantom.create( this.options.phantomArguments || DEFAULT_PHANTOM_ARGUMENTS, {
        binary: require('phantomjs').path,
        onExit: function() {
            _this.phantom = null;
            console.log('phantom crashed, restarting...')
            process.nextTick(_.bind(_this.createPhantom, _this));
        }
    }, _.bind(this.onPhantomCreate, this));
};

server.onPhantomCreate = function(phantom) {
    console.log('started phantom')
    this.phantom = phantom;
    this.phantom.id = Math.random().toString(36);
};

// Gets the URL to prerender from a request, stripping out unnecessary parts
function getUrl(req) {
    var decodedUrl
      , parts;

    try {
        decodedUrl = decodeURIComponent(req.url);
    } catch (e) {
        decodedUrl = req.url;
    }

    parts = url.parse(decodedUrl, true);

    // Remove the _escaped_fragment_ query parameter<commit_msg>making sure we are using different phantom processes with how cluster works with ports<commit_after>var phantom = require('phantom')
  , http = require('http')
  , url = require('url')
  , _ = require('lodash');

var DEFAULT_PHANTOM_ARGUMENTS = ['--load-images=false', '--ignore-ssl-errors=true'];

var DEFAULT_PHANTOM_BASE_PORT = 12300;

var PAGE_DONE_CHECK_TIMEOUT = process.env.PAGE_DONE_CHECK_TIMEOUT || 50;

var RESOURCE_DOWNLOAD_TIMEOUT = process.env.RESOURCE_DOWNLOAD_TIMEOUT || 10 * 1000;

var WAIT_AFTER_LAST_REQUEST = process.env.WAIT_AFTER_LAST_REQUEST || 500;

var JS_CHECK_TIMEOUT = process.env.JS_CHECK_TIMEOUT || 50;

var JS_TIMEOUT = process.env.JS_TIMEOUT || 10 * 1000;

var EVALUATE_JAVASCRIPT_CHECK_TIMEOUT = process.env.EVALUATE_JAVASCRIPT_CHECK_TIMEOUT || 50;

// Normalizes unimportant differences in URLs - e.g. ensures
// http://google.com/ and http://google.com normalize to the same string
function normalizeUrl(u) {
    return url.format(url.parse(u, true));
}

var server = exports = module.exports = {};

server.init = function(options) {
    this.plugins = this.plugins || [];
    this.options = options;

    return this;
};

server.start = function() {
    if(!this.options.isMaster) {
        this.createPhantom();
    }
};

server.use = function(plugin) {
    this.plugins.push(plugin);
    if (typeof plugin.init === 'function') plugin.init(this);
};

server._pluginEvent = function(methodName, args, callback) {
    var _this = this
      , index = 0
      , next;

    next = function() {
        var layer = _this.plugins[index++];
        if (!layer) return callback();

        var method = layer[methodName];

        if (method) {
            method.apply(layer, args);
        } else {
            next();
        }
    }
    
    args.push(next);
    next();
};

server.createPhantom = function() {
    var _this = this;
    console.log('starting phantom')

    var port = this.options.phantomBasePort || DEFAULT_PHANTOM_BASE_PORT;
    phantom.create( this.options.phantomArguments || DEFAULT_PHANTOM_ARGUMENTS, {
        binary: require('phantomjs').path,
        port: port + this.options.worker.id + 1,
        onExit: function() {
            _this.phantom = null;
            console.log('phantom crashed, restarting...')
            process.nextTick(_.bind(_this.createPhantom, _this));
        }
    }, _.bind(this.onPhantomCreate, this));
};

server.onPhantomCreate = function(phantom) {
    console.log('started phantom')
    this.phantom = phantom;
    this.phantom.id = Math.random().toString(36);
};

// Gets the URL to prerender from a request, stripping out unnecessary parts
function getUrl(req) {
    var decodedUrl
      , parts;

    try {
        decodedUrl = decodeURIComponent(req.url);
    } catch (e) {
        decodedUrl = req.url;
    }

    parts = url.parse(decodedUrl, true);

    // Remove the _escaped_fragment_ query parameter
{'fraction_non_alphanumeric': 0.13025512037369744, 'fraction_numerical': 0.009701760689902982, 'mean_word_length': 3.0694444444444446, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3251356', 'n_tokens_mistral': 1857, 'n_tokens_neox': 1734, 'n_words': 373}