<commit_before>require "socket"

module Logstasher

  HTTP_PARAMS_TO_STASH = [
    "HTTP_FROM", "HTTP_HOST", "HTTP_USER_AGENT", "HTTP_X_CLUSTER_CLIENT_IP",
    "HTTP_X_FORWARDED_FOR", "HTTP_X_FORWARDED_PROTO", "ORIGINAL_FULLPATH",
    "HTTP_ACCEPT_LANGUAGE", "HTTP_REFERER", "REMOTE_ADDR", "REQUEST_METHOD",
    "SERVER_ADDR", "CONTENT_LENGTH", "HTTP_ORIGIN", "HTTP_AUTHORIZATION",
    "HTTP_SSLSESSIONID", "X_MOBILE_DEVICE", "HTTP_X_COUNTRY_CODE" ]

  IP_PARAMS = [
    "HTTP_X_CLUSTER_CLIENT_IP", "HTTP_X_FORWARDED_FOR", "REMOTE_ADDR",
    "SERVER_ADDR", "clientip"
  ]

  def self.logger
    return if Rails.env.test?
    return @logger if @logger
    @logger = Logger.new("log/#{ Rails.env }.logstash.log")
    @logger.formatter = proc do |severity, datetime, progname, msg|
      # strings get written to the log file verbatim
      "#{ msg }\n"
    end
    @logger
  end

  def self.ip_from_request_env(request_env)
    # try a few params for IP. Proxies will shuffle around requester IP
    %w( HTTP_X_FORWARDED_FOR HTTP_X_CLUSTER_CLIENT_IP REMOTE_ADDR ).each do |param|
      return request_env[ param ] unless request_env[ param ].blank?
    end
    nil<commit_msg>Start tracking the Do Not Track HTTP header<commit_after>require "socket"

module Logstasher

  HTTP_PARAMS_TO_STASH = [
    "HTTP_FROM", "HTTP_HOST", "HTTP_USER_AGENT", "HTTP_X_CLUSTER_CLIENT_IP",
    "HTTP_X_FORWARDED_FOR", "HTTP_X_FORWARDED_PROTO", "ORIGINAL_FULLPATH",
    "HTTP_ACCEPT_LANGUAGE", "HTTP_REFERER", "REMOTE_ADDR", "REQUEST_METHOD",
    "SERVER_ADDR", "CONTENT_LENGTH", "HTTP_ORIGIN", "HTTP_AUTHORIZATION",
    "HTTP_SSLSESSIONID", "X_MOBILE_DEVICE", "HTTP_X_COUNTRY_CODE", "HTTP_DNT" ]

  IP_PARAMS = [
    "HTTP_X_CLUSTER_CLIENT_IP", "HTTP_X_FORWARDED_FOR", "REMOTE_ADDR",
    "SERVER_ADDR", "clientip"
  ]

  def self.logger
    return if Rails.env.test?
    return @logger if @logger
    @logger = Logger.new("log/#{ Rails.env }.logstash.log")
    @logger.formatter = proc do |severity, datetime, progname, msg|
      # strings get written to the log file verbatim
      "#{ msg }\n"
    end
    @logger
  end

  def self.ip_from_request_env(request_env)
    # try a few params for IP. Proxies will shuffle around requester IP
    %w( HTTP_X_FORWARDED_FOR HTTP_X_CLUSTER_CLIENT_IP REMOTE_ADDR ).each do |param|
      return request_env[ param ] unless request_env[ param ].blank?
    end
    nil
{'fraction_non_alphanumeric': 0.1108786610878661, 'fraction_numerical': 0.0, 'mean_word_length': 4.325167037861916, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4802627', 'n_tokens_mistral': 914, 'n_tokens_neox': 889, 'n_words': 191}