<commit_before>var phantom = require('phantom')
  , http = require('http')
  , url = require('url')
  , _ = require('lodash');

var prerender = exports = module.exports = {};

prerender.createServer = function() {
    this.plugins = this.plugins || [];

    this.createPhantom();
};

prerender.use = function(plugin) {
    this.plugins = this.plugins || [];

    this.plugins.push(plugin);
    if (typeof plugin.init === 'function') plugin.init();
};

prerender.createPhantom = function() {
    phantom.create('--load-images=false', { binary: require('phantomjs').path }, _.bind(this.onPhantomCreate, this));
};

prerender.onPhantomCreate = function(phantom) {
    this.phantom = phantom;

    this.createWebServer();
};

prerender.createWebServer = function() {
    var _this = this;

    http.createServer(_.bind(this.onRequest, this)).listen(process.env.PORT || 3000);
    console.log('Server running on port ' + (process.env.PORT || 3000));
};

prerender.onRequest = function(req, res) {
    var _this = this;
    res.send = _.bind(this.send, res, req);

    req.prerender = {
        url: this.getUrl(req).substr(1),
        start: new Date()
    };

    console.log('getting', req.prerender.url);

    this.pluginsBeforePhantomRequest(req, res, function(){

        _this.phantom.createPage(function(page){
            req.prerender.page = page;
            _this.onPhantomPageCreate(req, res);
        });
    });
};

prerender.getUrl = function(req) {
    if (req.url.indexOf('_escaped_fragment_') === -1) return req.url;

    var parts = url.parse(req.url, true);

    if(parts.query['_escaped_fragment_']) parts.hash = '#!' + parts.query['_escaped_fragment_'];
    delete parts.query['_escaped_fragment_'];
    delete parts.search;
<commit_msg>fixing query string check for _escaped_fragment_ in bad urls<commit_after>var phantom = require('phantom')
  , http = require('http')
  , url = require('url')
  , _ = require('lodash');

var prerender = exports = module.exports = {};

prerender.createServer = function() {
    this.plugins = this.plugins || [];

    this.createPhantom();
};

prerender.use = function(plugin) {
    this.plugins = this.plugins || [];

    this.plugins.push(plugin);
    if (typeof plugin.init === 'function') plugin.init();
};

prerender.createPhantom = function() {
    phantom.create('--load-images=false', { binary: require('phantomjs').path }, _.bind(this.onPhantomCreate, this));
};

prerender.onPhantomCreate = function(phantom) {
    this.phantom = phantom;

    this.createWebServer();
};

prerender.createWebServer = function() {
    var _this = this;

    http.createServer(_.bind(this.onRequest, this)).listen(process.env.PORT || 3000);
    console.log('Server running on port ' + (process.env.PORT || 3000));
};

prerender.onRequest = function(req, res) {
    var _this = this;
    res.send = _.bind(this.send, res, req);

    req.prerender = {
        url: this.getUrl(req).substr(1),
        start: new Date()
    };

    console.log('getting', req.prerender.url);

    this.pluginsBeforePhantomRequest(req, res, function(){

        _this.phantom.createPage(function(page){
            req.prerender.page = page;
            _this.onPhantomPageCreate(req, res);
        });
    });
};

prerender.getUrl = function(req) {
    var parts = url.parse(decodeURIComponent(req.url), true);

    if (!parts.query.hasOwnProperty('_escaped_fragment_')) return req.url;

    if(parts.query['_escaped_fragment_']) parts.hash = '#!' + parts.query['_escaped_fragment_'];
    delete parts.query['_escaped_fragment_'];
    delete parts.search;

{'fraction_non_alphanumeric': 0.16452513966480448, 'fraction_numerical': 0.005307262569832402, 'mean_word_length': 3.5559796437659035, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '606671', 'n_tokens_mistral': 1226, 'n_tokens_neox': 1123, 'n_words': 201}