<commit_before><?php

class Bouncer_Rules_Robot
{

    public static function load()
    {
        Bouncer::addRule('robot_identity', array('Bouncer_Rules_Robot', 'robot_identity'));
    }

    public static function robot_identity($identity)
    {
        $scores = array();
        $score = 0;

        if ($identity['type'] != Bouncer::ROBOT || empty($identity['name'])) {
            return null;
        }

        $addr = $identity['addr'];
        $host = $identity['host'];
        $headers = $identity['headers'];

        switch ($identity['name']) {
            // top crawlers
            case 'google':
                $score += strpos($host, 'googlebot.com') === false ? -5 : 1;
                $score += empty($headers['From']) ? -5 : 1;
                break;
            case 'mediapartners':
                $score += strpos($host, 'googlebot.com') === false ? -5 : 2.5;
                $score += $identity['fingerprint'] != '4<PASSWORD>' ? -5 : 2.5;
                break;
            case 'googlemobile':
                $score += strpos($host, 'googlebot.com') === false ? -5 : 2.5;
                $score += empty($headers['From']) ? -5 : 2.5;
                break;
            case 'yahoo':
                $score += strpos($host, 'yahoo.') === false && strpos($host, 'aliyun.com') === false ? -5 : 2.5;
                break;
            case 'msnbot':
                $score += strpos($host, 'msn.com') === false && empty($headers['From']) ? -5 : 1;
                break;
            case 'voila':
                $score += strpos($host, 'fti.net') === false && strpos($host, 'voilabot.orange.fr') === false ? -5 : 2.5;
                $score += $identity['fingerprint'] != '<KEY>' ? -5 : 2.5;
                break;
            case 'orange':
                $score += strpos($host, 'fti.net') === false ? -5 : 2.5;
                break;
            case 'naverbot':
                $score += (strpos($host, 'naver.jp') === false && strpos($addr, '61.247.204.') === false) ? -5 : 1;
                break;
            case 'scoutjet':
                $score += strpos($host, 'scoutjet.com') === false ? -5 : 2.5;
                $score += $identity['fingerprint'] != 'd970c6ffb8d5547d9f6052207200b0dd' ? -5 : 2.5;
                break;
            case 'baidu':
                $score += (strpos($host, 'baidu.')   === false
                        && strpos($addr, '119.63.')  === false
                        && strpos($addr, '123.125.') === false
                        && strpos($addr, '125.39.')  === false
                        && strpos($addr, '180.76.')  === false
                        && strpos($addr, '220.181.') === false) ? -5 : 2.5;
                break;
            case 'ask':
                $score += strpos($host, 'ask.com') === false ? -5 : 2.5;
                $score += $identity['fingerprint'] != '742b4a79a150f6ab785e558ddbf7aaa6' ? -5 : 2.5;
                break;
            case 'mailru':
                $score += strpos($host, 'mail.ru') === false ? -5 : 1;
                break;
            case 'twiceler':
                $score += strpos($host, 'cuil.com') === false ? -5 : 2.5;
                $score += $identity['fingerprint'] != 'b90c08ff5b5c8265dbfd9c6fa65f2e09' ? -5 : 2.5;
                break;
            case 'spinn3r':
                $score += strpos($host, 'spinn3r.com') === false && strpos($host, 'softlayer.com') === false ? -5 : 1;
                break;
            case 'picsearch':
                $score += strpos($host, 'picsearch.com') === false ? -5 : 2.5;
                break;
            case 'exabot':
                $score += strpos($host, 'exabot.com') === false ? -5 : 1;
                break;
            case 'entireweb':
                $score += strpos($host, 'entireweb.' === false) ? -2.5 : 1;
                $score += $identity['extension'] != 'se' ? -2.5 : 1;
                $score += empty($headers['From']) ? -2.5 : 1.5;
                $score += $identity['fingerprint'] != '<KEY>' ? -2.5 : 1.5;
                break;
            case 'alexa':
                $score += strpos($host, 'amazonaws.com') === false ? -5 : 1;
                $score += empty($headers['From']) ? -5 : 1;
                break;
            case 'daum':
                $score += $identity['fingerprint'] != '<KEY>' &&
                          $identity['fingerprint'] != '536cb87617e5a9ffdfae53365f116d89' ? -5 : 2.5;
                $score += $identity['extension'] != 'kr' ? -5 : 1;
                break;
            case 'soso':
                $score += (strpos($addr, '124.115.') === false && strpos($addr, '114.80.') === false) ? -5 : 2.5;
                break;
            case 'youdao':
                $score += strpos($addr, '61.135.') === false ? -5 : 2.5;
                break;
            case 'hatena':
                $score += $identity['fingerprint'] != '<KEY>' ? -5 : 1;
                $score += $identity['extension'] != 'jp' ? -5 : 1;
                break;
            case 'dotbot':
                $score += strpos($host, 'dotnetdotcom.org') === false ? -4 : 1;
                $score += $identity['extension'] != 'us' ? -3 : 1;
                $score += $identity['fingerprint'] != '1ba3e09e05c3a64578777e53d4f20a3c' ? -3 : 1;
                break;
            case 'sogou':
                $score += $identity['fingerprint'] != 'a<PASSWORD>' &&
                          $identity['fingerprint'] != 'a6c<PASSWORD>4<PASSWORD>3<PASSWORD>' ? -5 : 2.5;
                $score += $identity['extension'] != 'cn' ? -5 : 2.5;
                break;
            case 'alexa':
                $score += strpos($host, 'amazonaws.com') === false ? -5 : 2.5;
                $score += empty($headers['From']) ? -5 : 2.5;
                break;
            case 'setooz':
                $score += strpos($host, 'setooz.com') === false ? -5 : 2.5;
                break;
            case 'goo':
                $score += strpos($host, 'super-goo.com') === false ? -5 : 2.5;
                break;
            case 'mlbot':
                $score += (strpos($addr, '66.219.58.') === false && strpos($addr, '71.41.201.') === false) ? -5 : 2.5;
                break;
            case 'feedburner':
                $score += $identity['fingerprint'] != '<KEY>' &&
                          $identity['fingerprint'] != '84e14e474b5972e7b11fae97d08fff4c' ? -5 : 2.5;
                $score += $identity['extension'] != 'us' ? -5 : 2.5;
                break;
            case 'netcraft':
                $score += strpos($host, 'amazonaws.com') === false && strpos($host, 'netcraft.com') === false ? -5 : 2.5;
                // $score += $identity['fingerprint'] != '6<PASSWORD>' ? -5 : 2.5;
                break;
            case 'radian6':
                $score += strpos($addr, '142.166.') === false &&
                          strpos($addr, '207.34.25') === false &&
                          strpos($host, 'amazonaws.com') === false &&
                          strpos($host, 'softlayer.com') === false ? -5 : 2.5;
                break;
            case 'socialmention':
                $score += $identity['fingerprint'] != '<PASSWORD>' ? -5 : 2.5;
                break;
            case 'yandex':
                $score += strpos($host, 'yandex') === false ? -5 : 2.5;
                break;
            case 'friendfeed':
                $score += (strpos($host, 'facebook.com') === false && strpos($addr, '69.63.180.') === false) ? -5 : 2.5;
                break;
            case 'spbot':
                $score += strpos($host, 'amazonaws.com') === false ? -5 : 2.5;<commit_msg>friendfeed ips<commit_after><?php

class Bouncer_Rules_Robot
{

    public static function load()
    {
        Bouncer::addRule('robot_identity', array('Bouncer_Rules_Robot', 'robot_identity'));
    }

    public static function robot_identity($identity)
    {
        $scores = array();
        $score = 0;

        if ($identity['type'] != Bouncer::ROBOT || empty($identity['name'])) {
            return null;
        }

        $addr = $identity['addr'];
        $host = $identity['host'];
        $headers = $identity['headers'];

        switch ($identity['name']) {
            // top crawlers
            case 'google':
                $score += strpos($host, 'googlebot.com') === false ? -5 : 1;
                $score += empty($headers['From']) ? -5 : 1;
                break;
            case 'mediapartners':
                $score += strpos($host, 'googlebot.com') === false ? -5 : 2.5;
                $score += $identity['fingerprint'] != '4b8841489bb368a2e0defed8149bf912' ? -5 : 2.5;
                break;
            case 'googlemobile':
                $score += strpos($host, 'googlebot.com') === false ? -5 : 2.5;
                $score += empty($headers['From']) ? -5 : 2.5;
                break;
            case 'yahoo':
                $score += strpos($host, 'yahoo.') === false && strpos($host, 'aliyun.com') === false ? -5 : 2.5;
                break;
            case 'msnbot':
                $score += strpos($host, 'msn.com') === false && empty($headers['From']) ? -5 : 1;
                break;
            case 'voila':
                $score += strpos($host, 'fti.net') === false && strpos($host, 'voilabot.orange.fr') === false ? -5 : 2.5;
                $score += $identity['fingerprint'] != '<KEY>' ? -5 : 2.5;
                break;
            case 'orange':
                $score += strpos($host, 'fti.net') === false ? -5 : 2.5;
                break;
            case 'naverbot':
                $score += (strpos($host, 'naver.jp') === false && strpos($addr, '61.247.204.') === false) ? -5 : 1;
                break;
            case 'scoutjet':
                $score += strpos($host, 'scoutjet.com') === false ? -5 : 2.5;
                $score += $identity['fingerprint'] != '<KEY>' ? -5 : 2.5;
                break;
            case 'baidu':
                $score += (strpos($host, 'baidu.')   === false
                        && strpos($addr, '119.63.')  === false
                        && strpos($addr, '123.125.') === false
                        && strpos($addr, '125.39.')  === false
                        && strpos($addr, '180.76.')  === false
                        && strpos($addr, '220.181.') === false) ? -5 : 2.5;
                break;
            case 'ask':
                $score += strpos($host, 'ask.com') === false ? -5 : 2.5;
                $score += $identity['fingerprint'] != '742b4a79a150f6ab785e558ddbf7aaa6' ? -5 : 2.5;
                break;
            case 'mailru':
                $score += strpos($host, 'mail.ru') === false ? -5 : 1;
                break;
            case 'twiceler':
                $score += strpos($host, 'cuil.com') === false ? -5 : 2.5;
                $score += $identity['fingerprint'] != 'b<PASSWORD>' ? -5 : 2.5;
                break;
            case 'spinn3r':
                $score += strpos($host, 'spinn3r.com') === false && strpos($host, 'softlayer.com') === false ? -5 : 1;
                break;
            case 'picsearch':
                $score += strpos($host, 'picsearch.com') === false ? -5 : 2.5;
                break;
            case 'exabot':
                $score += strpos($host, 'exabot.com') === false ? -5 : 1;
                break;
            case 'entireweb':
                $score += strpos($host, 'entireweb.' === false) ? -2.5 : 1;
                $score += $identity['extension'] != 'se' ? -2.5 : 1;
                $score += empty($headers['From']) ? -2.5 : 1.5;
                $score += $identity['fingerprint'] != '<KEY>' ? -2.5 : 1.5;
                break;
            case 'alexa':
                $score += strpos($host, 'amazonaws.com') === false ? -5 : 1;
                $score += empty($headers['From']) ? -5 : 1;
                break;
            case 'daum':
                $score += $identity['fingerprint'] != '<KEY>' &&
                          $identity['fingerprint'] != '536cb87617e5a9ffdfae53365f116d89' ? -5 : 2.5;
                $score += $identity['extension'] != 'kr' ? -5 : 1;
                break;
            case 'soso':
                $score += (strpos($addr, '124.115.') === false && strpos($addr, '114.80.') === false) ? -5 : 2.5;
                break;
            case 'youdao':
                $score += strpos($addr, '61.135.') === false ? -5 : 2.5;
                break;
            case 'hatena':
                $score += $identity['fingerprint'] != '<KEY>' ? -5 : 1;
                $score += $identity['extension'] != 'jp' ? -5 : 1;
                break;
            case 'dotbot':
                $score += strpos($host, 'dotnetdotcom.org') === false ? -4 : 1;
                $score += $identity['extension'] != 'us' ? -3 : 1;
                $score += $identity['fingerprint'] != '1ba3e09e05c3a64578777e53d4f20a3c' ? -3 : 1;
                break;
            case 'sogou':
                $score += $identity['fingerprint'] != 'a86f74048055ff8ea8a8570615c478f4' &&
                          $identity['fingerprint'] != 'a6c9efa47da38d7946b4bb40d81f66ee' ? -5 : 2.5;
                $score += $identity['extension'] != 'cn' ? -5 : 2.5;
                break;
            case 'alexa':
                $score += strpos($host, 'amazonaws.com') === false ? -5 : 2.5;
                $score += empty($headers['From']) ? -5 : 2.5;
                break;
            case 'setooz':
                $score += strpos($host, 'setooz.com') === false ? -5 : 2.5;
                break;
            case 'goo':
                $score += strpos($host, 'super-goo.com') === false ? -5 : 2.5;
                break;
            case 'mlbot':
                $score += (strpos($addr, '66.219.58.') === false && strpos($addr, '71.41.201.') === false) ? -5 : 2.5;
                break;
            case 'feedburner':
                $score += $identity['fingerprint'] != 'cdcb<PASSWORD>d642' &&
                          $identity['fingerprint'] != '84e14e474b5972e7b11fae97d08fff4c' ? -5 : 2.5;
                $score += $identity['extension'] != 'us' ? -5 : 2.5;
                break;
            case 'netcraft':
                $score += strpos($host, 'amazonaws.com') === false && strpos($host, 'netcraft.com') === false ? -5 : 2.5;
                // $score += $identity['fingerprint'] != '6fdbdebe4a4e159db61b246974a63efb' ? -5 : 2.5;
                break;
            case 'radian6':
                $score += strpos($addr, '142.166.') === false &&
                          strpos($addr, '207.34.25') === false &&
                          strpos($host, 'amazonaws.com') === false &&
                          strpos($host, 'softlayer.com') === false ? -5 : 2.5;
                break;
            case 'socialmention':
                $score += $identity['fingerprint'] != 'a9b11c963519135d4b07c6b6ad36c0de' ? -5 : 2.5;
                break;
            case 'yandex':
                $score += strpos($host, 'yandex') === false ? -5 : 2.5;
                break;
            case 'friendfeed':
                $score += strpos($host, 'facebook.com') === false &&
                          strpos($addr, '69.63.180.') === false &&
                          strpos($addr, '69.171.233.') === false ? -5 : 2.5;
                break;
            case 'spbot':
                $score += strpos($host, 'amazonaws.com') === false ? -5 : 2.5;
{'fraction_non_alphanumeric': 0.18866449511400651, 'fraction_numerical': 0.05094462540716612, 'mean_word_length': 1.414438502673797, 'pattern_counts': {'":': 0, '<': 24, '<?xml version=': 0, '>': 22, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5851247', 'n_tokens_mistral': 5272, 'n_tokens_neox': 4681, 'n_words': 979}