<commit_before>var phridge = require('phridge'),
    redis = require("redis");

var htmlcleaner = require(__dirname+'/htmlcleaner'),
    translateSeo = require(__dirname+'/translate-seo');

var client = redis.createClient();


var bots = [
    'facebookexternalhit/1.1 (+http://www.facebook.com/externalhit_uatext.php)',
    'facebookexternalhit/1.1 (+http://www.facebook.com/externalhit_uatext.php)',
    'Facebot',
    'Googlebot'
];


module.exports = {
    middleware: function(req, res, next) {

        var fragment = req.query._escaped_fragment_,
            userAgent = req.headers['user-agent'];

        if(bots.indexOf(userAgent)==-1) {
            if(fragment===undefined||typeof fragment!='string') {
                return next();
            }
        }
<commit_msg>More bots for SEO<commit_after>var phridge = require('phridge'),
    redis = require("redis");

var htmlcleaner = require(__dirname+'/htmlcleaner'),
    translateSeo = require(__dirname+'/translate-seo');

var client = redis.createClient();


var bots = [
    'facebookexternalhit/1.1 (+http://www.facebook.com/externalhit_uatext.php)',
    'facebookexternalhit/1.1 (+http://www.facebook.com/externalhit_uatext.php)',
    'Facebot',
    'Googlebot',
    'Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)',
    'Googlebot/2.1 (+http://www.google.com/bot.html)',
    'Googlebot-News',
    'Googlebot-Image/1.0',
    'Googlebot-Video/1.0',
    '(compatible; Googlebot-Mobile/2.1; +http://www.google.com/bot.html)',
    '(compatible; Mediapartners-Google/2.1; +http://www.google.com/bot.html)',
    'Mediapartners-Google',
    'AdsBot-Google (+http://www.google.com/adsbot.html)',
    'Mozilla/5.0 (compatible; bingbot/2.0; +http://www.bing.com/bingbot.htm)',
    'Mozilla/5.0 (iPhone; CPU iPhone OS 7_0 like Mac OS X) AppleWebKit/537.51.1 (KHTML, like Gecko) Version/7.0 Mobile/11A465 Safari/9537.53 (compatible; bingbot/2.0; +http://www.bing.com/bingbot.htm)',
    'Mozilla/5.0 (Windows Phone 8.1; ARM; Trident/7.0; Touch; rv:11.0; IEMobile/11.0; NOKIA; Lumia 530) like Gecko (compatible; bingbot/2.0; +http://www.bing.com/bingbot.htm)'
];


module.exports = {
    middleware: function(req, res, next) {

        var fragment = req.query._escaped_fragment_,
            userAgent = req.headers['user-agent'];

        if(bots.indexOf(userAgent)==-1) {
            if(fragment===undefined||typeof fragment!='string') {
                return next();
            }
        }

{'fraction_non_alphanumeric': 0.17649434571890146, 'fraction_numerical': 0.02827140549273021, 'mean_word_length': 3.198305084745763, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 12, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '741755', 'n_tokens_mistral': 902, 'n_tokens_neox': 862, 'n_words': 134}