<commit_before># If Mini Profiler is included via gem
if Rails.configuration.respond_to?(:load_mini_profiler) && Rails.configuration.load_mini_profiler
  require 'rack-mini-profiler'
  require 'flamegraph'

  begin
    require 'memory_profiler' if RUBY_VERSION >= "2.1.0"
  rescue => e
     STDERR.put "#{e} failed to require mini profiler"
  end

  # initialization is skipped so trigger it
  Rack::MiniProfilerRails.initialize!(Rails.application)
end

if defined?(Rack::MiniProfiler)

  # note, we may want to add some extra security here that disables mini profiler in a multi hosted env unless user global admin
  #   raw_connection means results are not namespaced
  #
  # namespacing gets complex, cause mini profiler is in the rack chain way before multisite
  Rack::MiniProfiler.config.storage_instance = Rack::MiniProfiler::RedisStore.new(connection:  DiscourseRedis.raw_connection)

  skip = [
    /^\/message-bus/,
    /topics\/timings/,
    /assets/,
    /\/user_avatar\//,
    /\/letter_avatar\//,
    /\/highlight-js\//,
    /qunit/,
    /srv\/status/,
    /commits-widget/,
    /^\/cdn_asset/,
    /^\/logs/,
    /^\/site_customizations/,
    /^\/uploads/,
    /^\/javascripts\//,
    /^\/images\//,
    /^\/stylesheets\//
  ]

  # For our app, let's just show mini profiler always, polling is chatty so nuke that
  Rack::MiniProfiler.config.pre_authorize_cb = lambda do |env|
    path = env['PATH_INFO']

    (env['HTTP_USER_AGENT'] !~ /iPad|iPhone|Nexus 7|Android/) &&
    !skip.any?{|re| re =~ path}
  end

  # without a user provider our results will use the ip address for namespacing
  #  with a load balancer in front this becomes really bad as some results can
  #  be stored associated with ip1 as the user and retrieved using ip2 causing 404s
  Rack::MiniProfiler.config.user_provider = lambda do |env|
    request = Rack::Request.new(env)
    id = request.cookies["_t"] || request.ip || "unknown"
    id = id.to_s
    # some security, lets not have these tokens floating about
    Digest::MD5.hexdigest(id)
  end

  Rack::MiniProfiler.config.position = 'left'
  Rack::MiniProfiler.config.backtrace_ignores ||= []
  Rack::MiniProfiler.config.backtrace_ignores << /lib\/rack\/message_bus.rb/
  Rack::MiniProfiler.config.backtrace_ignores << /config\/initializers\/silence_logger/
  Rack::MiniProfiler.config.backtrace_ignores << /config\/initializers\/quiet_logger/


  # Rack::MiniProfiler.counter_method(ActiveRecord::QueryMethods, 'build_arel')
  # Rack::MiniProfiler.counter_method(Array, 'uniq')
  # require "#{Rails.root}/vendor/backports/notification"

  # inst = Class.new
  # class << inst
  #   def start(name,id,payload)
  #     if Rack::MiniProfiler.current && name !~ /(process_action.action_controller)|(render_template.action_view)/
  #       @prf ||= {}
  #       @prf[id] ||= []
  #       @prf[id] << Rack::MiniProfiler.start_step("#{payload[:serializer] if name =~ /serialize.serializer/} #{name}")
  #     end
  #   end

  #   def finish(name,id,payload)
  #     if Rack::MiniProfiler.current && name !~ /(process_action.action_controller)|(render_template.action_view)/
  #       t = @prf[id].pop
  #       @prf.delete id unless t
  #       Rack::MiniProfiler.finish_step t
  #     end
  #   end
  # end
  # disabling for now cause this slows stuff down too much
  # ActiveSupport::Notifications.subscribe(/.*/, inst)

  # Rack::MiniProfiler.profile_method ActionView::PathResolver, 'find_templates'
end<commit_msg>add some extra diagnostics<commit_after># If Mini Profiler is included via gem
if Rails.configuration.respond_to?(:load_mini_profiler) && Rails.configuration.load_mini_profiler
  require 'rack-mini-profiler'
  require 'flamegraph'

  begin
    require 'memory_profiler' if RUBY_VERSION >= "2.1.0"
  rescue => e
     STDERR.put "#{e} failed to require mini profiler"
  end

  # initialization is skipped so trigger it
  Rack::MiniProfilerRails.initialize!(Rails.application)
end

if defined?(Rack::MiniProfiler)

  # note, we may want to add some extra security here that disables mini profiler in a multi hosted env unless user global admin
  #   raw_connection means results are not namespaced
  #
  # namespacing gets complex, cause mini profiler is in the rack chain way before multisite
  Rack::MiniProfiler.config.storage_instance = Rack::MiniProfiler::RedisStore.new(connection:  DiscourseRedis.raw_connection)

  skip = [
    /^\/message-bus/,
    /topics\/timings/,
    /assets/,
    /\/user_avatar\//,
    /\/letter_avatar\//,
    /\/highlight-js\//,
    /qunit/,
    /srv\/status/,
    /commits-widget/,
    /^\/cdn_asset/,
    /^\/logs/,
    /^\/site_customizations/,
    /^\/uploads/,
    /^\/javascripts\//,
    /^\/images\//,
    /^\/stylesheets\//
  ]

  # For our app, let's just show mini profiler always, polling is chatty so nuke that
  Rack::MiniProfiler.config.pre_authorize_cb = lambda do |env|
    path = env['PATH_INFO']

    (env['HTTP_USER_AGENT'] !~ /iPad|iPhone|Nexus 7|Android/) &&
    !skip.any?{|re| re =~ path}
  end

  # without a user provider our results will use the ip address for namespacing
  #  with a load balancer in front this becomes really bad as some results can
  #  be stored associated with ip1 as the user and retrieved using ip2 causing 404s
  Rack::MiniProfiler.config.user_provider = lambda do |env|
    request = Rack::Request.new(env)
    id = request.cookies["_t"] || request.ip || "unknown"
    id = id.to_s
    # some security, lets not have these tokens floating about
    Digest::MD5.hexdigest(id)
  end

  Rack::MiniProfiler.config.position = 'left'
  Rack::MiniProfiler.config.backtrace_ignores ||= []
  Rack::MiniProfiler.config.backtrace_ignores << /lib\/rack\/message_bus.rb/
  Rack::MiniProfiler.config.backtrace_ignores << /config\/initializers\/silence_logger/
  Rack::MiniProfiler.config.backtrace_ignores << /config\/initializers\/quiet_logger/


  # Rack::MiniProfiler.counter_method(ActiveRecord::QueryMethods, 'build_arel')
  # Rack::MiniProfiler.counter_method(Array, 'uniq')
  # require "#{Rails.root}/vendor/backports/notification"

  # inst = Class.new
  # class << inst
  #   def start(name,id,payload)
  #     if Rack::MiniProfiler.current && name !~ /(process_action.action_controller)|(render_template.action_view)/
  #       @prf ||= {}
  #       @prf[id] ||= []
  #       @prf[id] << Rack::MiniProfiler.start_step("#{payload[:serializer] if name =~ /serialize.serializer/} #{name}")
  #     end
  #   end

  #   def finish(name,id,payload)
  #     if Rack::MiniProfiler.current && name !~ /(process_action.action_controller)|(render_template.action_view)/
  #       t = @prf[id].pop
  #       @prf.delete id unless t
  #       Rack::MiniProfiler.finish_step t
  #     end
  #   end
  # end
  # disabling for now cause this slows stuff down too much
  # ActiveSupport::Notifications.subscribe(/.*/, inst)

  # Rack::MiniProfiler.profile_method ActionView::PathResolver, 'find_templates'
end


if ENV["PRINT_EXCEPTIONS"]
  trace      = TracePoint.new(:raise) do |tp|
    puts tp.raised_exception
    puts tp.raised_exception.backtrace.join("\n")
    puts
  end
  trace.enable
end
{'fraction_non_alphanumeric': 0.13246242449782272, 'fraction_numerical': 0.0028093833403567917, 'mean_word_length': 3.447220487195503, 'pattern_counts': {'":': 0, '<': 23, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '397249', 'n_tokens_mistral': 2470, 'n_tokens_neox': 2434, 'n_words': 544}