<commit_before>require "socket"

module Logstasher

  HTTP_PARAMS_TO_STASH = [
    "HTTP_FROM", "HTTP_HOST", "HTTP_USER_AGENT", "HTTP_X_CLUSTER_CLIENT_IP",
    "HTTP_X_FORWARDED_FOR", "HTTP_X_FORWARDED_PROTO", "ORIGINAL_FULLPATH",
    "HTTP_ACCEPT_LANGUAGE", "HTTP_REFERER", "REMOTE_ADDR", "REQUEST_METHOD",
    "SERVER_ADDR", "CONTENT_LENGTH", "HTTP_ORIGIN", "HTTP_AUTHORIZATION",
    "HTTP_SSLSESSIONID", "X_MOBILE_DEVICE" ]

  IP_PARAMS = [
    "HTTP_X_CLUSTER_CLIENT_IP", "HTTP_X_FORWARDED_FOR", "REMOTE_ADDR",
    "SERVER_ADDR", "clientip"
  ]

  def self.logger
    return if Rails.env.test?
    return @logger if @logger
    @logger = Logger.new("log/#{ Rails.env }.logstash.log")
    @logger.formatter = proc do |severity, datetime, progname, msg|
      # strings get written to the log file verbatim
      "#{ msg }\n"
    end
    @logger
  end

  def self.ip_from_request_env(request_env)
    # try a few params for IP. Proxies will shuffle around requester IP
    %w( HTTP_X_FORWARDED_FOR HTTP_X_CLUSTER_CLIENT_IP REMOTE_ADDR ).each do |param|
      return request_env[ param ] unless request_env[ param ].blank?
    end
    nil
  end

  def self.payload_from_request(request)
    return { } unless request.is_a?(ActionDispatch::Request)
    payload = { }
    # grab nearly all the HTTP params from request.env
    payload.merge!(request.env.select{ |k,v|
      HTTP_PARAMS_TO_STASH.include?(k) && !v.blank? })
    # cleanup multiple IPs
    payload = Logstasher.split_multiple_ips(payload)
    payload[:clientip] = Logstasher.ip_from_request_env(payload)
    if request.env["HTTP_ACCEPT_LANGUAGE"]
      # there may be multiple variations of languages, plus other junk
      payload[:http_languages] = request.env["HTTP_ACCEPT_LANGUAGE"].
        split(/[;,]/).select{ |l| l =~ /^[a-z-]+$/i }.map(&:downcase).first
    end
    payload[:ssl] = request.ssl?.to_s
    payload[:bot] = Logstasher.is_user_agent_a_bot?(request.user_agent)
    # this can be overwritten by merging Logstasher.payload_from_user
    payload[:logged_in] = false
    payload[:i18n_locale] = I18n.locale.to_s.downcase
    payload[:http_locale_matches_i18n] = payload[:i18n_locale] == payload[:http_languages]
    payload[:http_lang_matches_i18n] = payload[:i18n_locale] &&
      payload[:http_languages] &&
      payload[:i18n_locale].split( "-" )[0] == payload[:http_languages].split( "-" )[0]
    payload
  end

  def self.payload_from_session(session)
    return { } unless session.is_a?(ActionDispatch::Request::Session)
    { session: session.to_hash.select{ |k,v|
      [ :session_id, :_csrf_token ].include?(k.to_sym) } }
  end

  def self.payload_from_user(user)
    return { } unless user.is_a?(User)
    { user_id: user.id,
      user_name: user.login,
      logged_in: true }
  end
<commit_msg>add Via header to logstash log<commit_after>require "socket"

module Logstasher

  HTTP_PARAMS_TO_STASH = [
    "HTTP_FROM", "HTTP_HOST", "HTTP_USER_AGENT", "HTTP_X_CLUSTER_CLIENT_IP",
    "HTTP_X_FORWARDED_FOR", "HTTP_X_FORWARDED_PROTO", "ORIGINAL_FULLPATH",
    "HTTP_ACCEPT_LANGUAGE", "HTTP_REFERER", "REMOTE_ADDR", "REQUEST_METHOD",
    "SERVER_ADDR", "CONTENT_LENGTH", "HTTP_ORIGIN", "HTTP_AUTHORIZATION",
    "HTTP_SSLSESSIONID", "X_MOBILE_DEVICE" ]

  IP_PARAMS = [
    "HTTP_X_CLUSTER_CLIENT_IP", "HTTP_X_FORWARDED_FOR", "REMOTE_ADDR",
    "SERVER_ADDR", "clientip"
  ]

  def self.logger
    return if Rails.env.test?
    return @logger if @logger
    @logger = Logger.new("log/#{ Rails.env }.logstash.log")
    @logger.formatter = proc do |severity, datetime, progname, msg|
      # strings get written to the log file verbatim
      "#{ msg }\n"
    end
    @logger
  end

  def self.ip_from_request_env(request_env)
    # try a few params for IP. Proxies will shuffle around requester IP
    %w( HTTP_X_FORWARDED_FOR HTTP_X_CLUSTER_CLIENT_IP REMOTE_ADDR ).each do |param|
      return request_env[ param ] unless request_env[ param ].blank?
    end
    nil
  end

  def self.payload_from_request(request)
    return { } unless request.is_a?(ActionDispatch::Request)
    payload = { }
    # grab nearly all the HTTP params from request.env
    payload.merge!(request.env.select{ |k,v|
      HTTP_PARAMS_TO_STASH.include?(k) && !v.blank? })
    # cleanup multiple IPs
    payload = Logstasher.split_multiple_ips(payload)
    payload[:clientip] = Logstasher.ip_from_request_env(payload)
    if request.env["HTTP_ACCEPT_LANGUAGE"]
      # there may be multiple variations of languages, plus other junk
      payload[:http_languages] = request.env["HTTP_ACCEPT_LANGUAGE"].
        split(/[;,]/).select{ |l| l =~ /^[a-z-]+$/i }.map(&:downcase).first
    end
    payload[:Via] = request.headers["Via"]
    payload[:ssl] = request.ssl?.to_s
    payload[:bot] = Logstasher.is_user_agent_a_bot?(request.user_agent)
    # this can be overwritten by merging Logstasher.payload_from_user
    payload[:logged_in] = false
    payload[:i18n_locale] = I18n.locale.to_s.downcase
    payload[:http_locale_matches_i18n] = payload[:i18n_locale] == payload[:http_languages]
    payload[:http_lang_matches_i18n] = payload[:i18n_locale] &&
      payload[:http_languages] &&
      payload[:i18n_locale].split( "-" )[0] == payload[:http_languages].split( "-" )[0]
    payload
  end

  def self.payload_from_session(session)
    return { } unless session.is_a?(ActionDispatch::Request::Session)
    { session: session.to_hash.select{ |k,v|
      [ :session_id, :_csrf_token ].include?(k.to_sym) } }
  end

  def self.payload_from_user(user)
    return { } unless user.is_a?(User)
    { user_id: user.id,
      user_name: user.login,
      logged_in: true }
  end

{'fraction_non_alphanumeric': 0.12686302342086586, 'fraction_numerical': 0.0056777856635912, 'mean_word_length': 4.015124555160142, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6698221', 'n_tokens_mistral': 2068, 'n_tokens_neox': 2028, 'n_words': 383}