<commit_before>module Split
  class Configuration
    BOTS = {
      'Baidu' => 'Chinese search engine',
      'Sogou' => 'Chinese search engine',
      'Gigabot' => 'Gigabot spider',
      'Googlebot' => 'Google spider',
      'libwww-perl' => 'Perl client-server library loved by script kids',
      'lwp-trivial' => 'Another Perl library loved by script kids',
      'msnbot' => 'Microsoft bot',
      'SiteUptime' => 'Site monitoring services',
      'Slurp' => 'Yahoo spider',
      'WordPress' => 'WordPress spider',
      'ZIBB' => 'ZIBB spider',
      'ZyBorg' => 'Zyborg? Hmmm....',
      "AdsBot-Google" => 'Google Adwords',
      "DigitalPersona Fingerprint Software" => 'HP Fingerprint scanner',
      "EventMachine HttpClient" => 'Ruby http library',
      "Go http package" => 'Go http library',
      "Python-urllib" => 'Python http library',
      "PycURL" => 'Python http library',
      "ShowyouBot" => 'Showyou Bot',
      "Test Certificate Info" => 'C http library?',
      "Wget" => 'wget unix CLI http client',
      "bitlybot" => 'bit.ly bot',
      "curl" => 'curl unix CLI http client',
      "facebookexternalhit" => 'facebook bot',
      "spider" => 'generic web spider',
      'check_http' => 'Nagios monitor',
      'NewRelicPinger' => 'NewRelic monitor',
      "Pingdom" => 'Pingdom monitoring',
      'Apache-HttpClient' => 'Java http library',
      'Java' => 'Generic Java http library',
      'ColdFusion' => 'ColdFusion http library',
      'rogerbot' => 'SeoMoz spider',
      'LongURL' => 'URL expander service',
      'UnwindFetch' => 'Gnip URL expander',
      'awe.sm' => 'Awe.sm URL expander',
      'Twitterbot' => 'Twitter URL expander',
      'Panopta' => 'Monitoring service',
      'AppEngine-Google' => 'Google App Engine',
    }
    attr_accessor :robot_regex
    attr_accessor :ignore_ip_addresses
    attr_accessor :db_failover
    attr_accessor :db_failover_on_db_error
    attr_accessor :db_failover_allow_parameter_override
    attr_accessor :allow_multiple_experiments
    attr_accessor :enabled
    attr_accessor :experiments
    attr_accessor :persistence
    attr_accessor :algorithm

    def disabled?
      !enabled
    end

    def experiment_for(name)
      if normalized_experiments
        # TODO symbols
        normalized_experiments[name.to_sym]
      end
    end

    def metrics
      return @metrics if defined?(@metrics)
      @metrics = {}
      if self.experiments
        self.experiments.each do |key, value|<commit_msg>Categorize & alphabetize bot list<commit_after>module Split
  class Configuration
    BOTS = {
      # Indexers
      "AdsBot-Google" => 'Google Adwords',
      'Baidu' => 'Chinese search engine',
      'Gigabot' => 'Gigabot spider',
      'Googlebot' => 'Google spider',
      'msnbot' => 'Microsoft bot',
      'rogerbot' => 'SeoMoz spider',
      'Slurp' => 'Yahoo spider',
      'Sogou' => 'Chinese search engine',
      "spider" => 'generic web spider',
      'WordPress' => 'WordPress spider',
      'ZIBB' => 'ZIBB spider',
      # HTTP libraries
      'Apache-HttpClient' => 'Java http library',
      'AppEngine-Google' => 'Google App Engine',
      "curl" => 'curl unix CLI http client',
      'ColdFusion' => 'ColdFusion http library',
      "EventMachine HttpClient" => 'Ruby http library',
      "Go http package" => 'Go http library',
      'Java' => 'Generic Java http library',
      'libwww-perl' => 'Perl client-server library loved by script kids',
      'lwp-trivial' => 'Another Perl library loved by script kids',
      "Python-urllib" => 'Python http library',
      "PycURL" => 'Python http library',
      "Test Certificate Info" => 'C http library?',
      "Wget" => 'wget unix CLI http client',
      # URL expanders / previewers
      'awe.sm' => 'Awe.sm URL expander',
      "bitlybot" => 'bit.ly bot',
      "facebookexternalhit" => 'facebook bot',
      'LongURL' => 'URL expander service',
      'Twitterbot' => 'Twitter URL expander',
      'UnwindFetch' => 'Gnip URL expander',
      # Uptime monitoring
      'check_http' => 'Nagios monitor',
      'NewRelicPinger' => 'NewRelic monitor',
      'Panopta' => 'Monitoring service',
      "Pingdom" => 'Pingdom monitoring',
      'SiteUptime' => 'Site monitoring services',
      # ???
      "DigitalPersona Fingerprint Software" => 'HP Fingerprint scanner',
      "ShowyouBot" => 'Showyou iOS app spider',
      'ZyBorg' => 'Zyborg? Hmmm....',
    }
    attr_accessor :robot_regex
    attr_accessor :ignore_ip_addresses
    attr_accessor :db_failover
    attr_accessor :db_failover_on_db_error
    attr_accessor :db_failover_allow_parameter_override
    attr_accessor :allow_multiple_experiments
    attr_accessor :enabled
    attr_accessor :experiments
    attr_accessor :persistence
    attr_accessor :algorithm

    def disabled?
      !enabled
    end

    def experiment_for(name)
      if normalized_experiments
        # TODO symbols
        normalized_experiments[name.to_sym]
      end
    end

    def metrics
      return @metrics if defined?(@metrics)
      @metrics = {}
      if self.experiments
        self.experiments.each do |key, value|
{'fraction_non_alphanumeric': 0.1270459859703819, 'fraction_numerical': 0.0, 'mean_word_length': 2.9153318077803205, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 79, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 1, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6929122', 'n_tokens_mistral': 1677, 'n_tokens_neox': 1608, 'n_words': 426}