<commit_before>require 'ipaddr'
require 'set'
require 'active_support'
require 'active_record'
require 'honeypot/ipaddr_ext'
require 'honeypot/remote_request'
require 'honeypot/remote_host'
require 'honeypot/rack'

require 'honeypot/railtie' if defined?(Rails::Railtie)

module Honeypot
  def self.included(base)
    base.class_eval do
      has_many :remote_requests, :as => :requestable, :dependent => :destroy
      has_many :remote_hosts, :through => :remote_requests, :uniq => true
    end
  end
  
  def log_remote_request(session, request)
    return unless session['honeypot.last_known_remote_ip'].present?
    remote_host = RemoteHost.find_or_create_by_ip_address session['honeypot.last_known_remote_ip']
    remote_request = remote_requests.find_or_create_by_remote_host_id remote_host.id
    remote_request.last_http_referer = request.referer if request.referer.present?
    remote_request.last_request_uri = request.request_uri if request.request_uri.present?
    remote_request.increment :hits
    remote_request.save!
    true
  end
  
  def related_requestables(seen_remote_host_ids = [])
    set = Set.new
    conditions = seen_remote_host_ids.present? ? [ "remote_hosts.id NOT IN (?)", seen_remote_host_ids ] : nil
    remote_hosts.scoped(:conditions => conditions).find_in_batches do |batch|
      batch.each do |remote_host|
        seen_remote_host_ids << remote_host.id
        remote_host.remote_requests.all(:include => :requestable).each do |remote_request|
          set << remote_request.requestable
        end
      end
    end
    if respond_to?(:actor) and actor != self
      set += actor.related_requestables(seen_remote_host_ids)
    end
    set.delete self
    set
  end
end<commit_msg>Save all requests, even if we know they're local/private.<commit_after>require 'ipaddr'
require 'set'
require 'active_support'
require 'active_record'
require 'honeypot/ipaddr_ext'
require 'honeypot/remote_request'
require 'honeypot/remote_host'
require 'honeypot/rack'

require 'honeypot/railtie' if defined?(Rails::Railtie)

module Honeypot
  def self.included(base)
    base.class_eval do
      has_many :remote_requests, :as => :requestable, :dependent => :destroy
      has_many :remote_hosts, :through => :remote_requests, :uniq => true
    end
  end
  
  def log_remote_request(session, request)
    effective_ip_address = session['honeypot.last_known_remote_ip'].present? ? session['honeypot.last_known_remote_ip'] : request.remote_ip
    remote_host = RemoteHost.find_or_create_by_ip_address effective_ip_address
    remote_request = remote_requests.find_or_create_by_remote_host_id remote_host.id
    remote_request.last_http_referer = request.referer if request.referer.present?
    remote_request.last_request_uri = request.request_uri if request.request_uri.present?
    remote_request.increment :hits
    remote_request.save!
    true
  end
  
  def related_requestables(seen_remote_host_ids = [])
    set = Set.new
    conditions = seen_remote_host_ids.present? ? [ "remote_hosts.id NOT IN (?)", seen_remote_host_ids ] : nil
    remote_hosts.scoped(:conditions => conditions).find_in_batches do |batch|
      batch.each do |remote_host|
        seen_remote_host_ids << remote_host.id
        remote_host.remote_requests.all(:include => :requestable).each do |remote_request|
          set << remote_request.requestable
        end
      end
    end
    if respond_to?(:actor) and actor != self
      set += actor.related_requestables(seen_remote_host_ids)
    end
    set.delete self
    set
  end
end
{'fraction_non_alphanumeric': 0.08936651583710407, 'fraction_numerical': 0.0, 'mean_word_length': 4.375379939209727, 'pattern_counts': {'":': 0, '<': 11, '<?xml version=': 0, '>': 15, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3167094', 'n_tokens_mistral': 1160, 'n_tokens_neox': 1134, 'n_words': 222}