<commit_before># ActiveRecordProfiler

module ActiveRecordProfiler
  require 'fileutils'
  require 'json'
  begin
    require 'fastercsv'
  rescue Exception => e
    RAILS_DEFAULT_LOGGER.debug("FasterCSV not available for use in ActiveRecordProfiler")
  end
  
  DURATION = 0
  COUNT = 1
  LONGEST = 2
  LONGEST_SQL = 3
  LOCATION = -1
  AVG_DURATION = -2
  
  DATE_FORMAT = '%Y-%m-%d'
  HOUR_FORMAT = '-%H'
  DATETIME_FORMAT = DATE_FORMAT + HOUR_FORMAT + '-%M'
  AGGREGATE_QUIET_PERIOD = 1.minutes
  
  CSV_DURATION = 0
  CSV_COUNT = 1
  CSV_AVG = 2
  CSV_LONGEST = 3
  CSV_LOCATION = 4
  CSV_LONGEST_SQL = 5
  
  class Collector
    # You can disable the profiler by setting 
    # ActiveRecordProfiler::Collector.profile_environments = []
    # (default: [ 'development', 'staging' ])
    cattr_accessor :profile_environments
    self.profile_environments = %w( development staging )
    
    cattr_accessor :profiler_enabled
    self.profiler_enabled = true
    
    # You can control the (approximate) frequency of statistics flushes by setting
    # ActiveRecordProfiler::Collector.stats_flush_period = 1.hour
    cattr_accessor :stats_flush_period
    self.stats_flush_period = 1.hour
    
    # Directory where profile data is recorded
    cattr_accessor :profile_dir
    self.profile_dir = File.join(RAILS_ROOT, "log", "profiler_data")

    # Any SQL statements matching this pattern will not be tracked by the profiler output
    # (though it will still appear in the enhanced SQL logging).
    cattr_accessor :sql_ignore_pattern
    self.sql_ignore_pattern = /^(SHOW FIELDS |SET SQL_AUTO_IS_NULL|SET NAMES |EXPLAIN |BEGIN|COMMIT)/
    
    cattr_accessor :app_path_pattern
    self.app_path_pattern = Regexp.new(Regexp.quote("/app/"))
    
    cattr_accessor :trim_root_path
    self.trim_root_path = RAILS_ROOT[-1] == "/" ? RAILS_ROOT : RAILS_ROOT + "/"
    
    cattr_accessor :storage_backend
    self.storage_backend = :json    # or :fastercsv
    
    cattr_accessor :profile_self
    self.profile_self = false

    attr_accessor :last_stats_flush
    attr_accessor :query_sites
    attr_accessor :profile_data_directory
    
    @@collector = nil
    def self.instance
      @@collector ||= Collector.new
    end
    
    def self.profiler_enabled?
      profile_environments.include?(ENV['RAILS_ENV'])
    end
    
    def self.profile_self?
      self.profile_self
    end
    
    def self.clear_data
      dir = Dir.new(profile_dir)
      prof_files = dir.entries.select{ |filename| /.prof$/.match(filename) }.map{ |filename| File.join(dir.path, filename) }
      FileUtils.rm(prof_files) if prof_files.size > 0
    end

    def initialize
      @query_sites = {}
      @last_stats_flush = nil
      @profile_data_directory = self.class.profile_dir
    end
    
    def call_location_name
      find_app_call_location || 'Non-application code'
    end
    
    def record_caller_info(location, seconds, sql)
      return if sql_ignore_pattern.match(sql)
      
      update_counts(location, seconds, 1, sql)
    end
    
    def record_self_info(seconds, name)
      record_caller_info(trim_location(caller.first), seconds, name)
    end
    
    def should_flush_stats?
      self.last_stats_flush ||= Time.now
      return(Time.now > self.last_stats_flush + stats_flush_period)
    end
    
    def flush_query_sites_statistics
      pid = $$
      flush_time = Time.now
      site_count = self.query_sites.keys.size
      RAILS_DEFAULT_LOGGER.info("Flushing ActiveRecordProfiler statistics for PID #{pid} at #{flush_time} (#{site_count} sites).")
      
      if (site_count > 0)
        FileUtils.makedirs(self.profile_data_directory)
      
        filename = File.join(self.profile_data_directory, "#{flush_time.strftime(DATETIME_FORMAT)}.#{pid}.prof")
        write_file(filename)
        <commit_msg>Adding thread id to profiler output file name and moving collector instance from class variable to thread-local variable. <commit_after># ActiveRecordProfiler

module ActiveRecordProfiler
  require 'fileutils'
  require 'json'
  begin
    require 'fastercsv'
  rescue Exception => e
    RAILS_DEFAULT_LOGGER.debug("FasterCSV not available for use in ActiveRecordProfiler")
  end
  
  DURATION = 0
  COUNT = 1
  LONGEST = 2
  LONGEST_SQL = 3
  LOCATION = -1
  AVG_DURATION = -2
  
  DATE_FORMAT = '%Y-%m-%d'
  HOUR_FORMAT = '-%H'
  DATETIME_FORMAT = DATE_FORMAT + HOUR_FORMAT + '-%M'
  AGGREGATE_QUIET_PERIOD = 1.minutes
  
  CSV_DURATION = 0
  CSV_COUNT = 1
  CSV_AVG = 2
  CSV_LONGEST = 3
  CSV_LOCATION = 4
  CSV_LONGEST_SQL = 5
  
  class Collector
    # You can disable the profiler by setting 
    # ActiveRecordProfiler::Collector.profile_environments = []
    # (default: [ 'development', 'staging' ])
    cattr_accessor :profile_environments
    self.profile_environments = %w( development staging )
    
    cattr_accessor :profiler_enabled
    self.profiler_enabled = true
    
    # You can control the (approximate) frequency of statistics flushes by setting
    # ActiveRecordProfiler::Collector.stats_flush_period = 1.hour
    cattr_accessor :stats_flush_period
    self.stats_flush_period = 1.hour
    
    # Directory where profile data is recorded
    cattr_accessor :profile_dir
    self.profile_dir = File.join(RAILS_ROOT, "log", "profiler_data")

    # Any SQL statements matching this pattern will not be tracked by the profiler output
    # (though it will still appear in the enhanced SQL logging).
    cattr_accessor :sql_ignore_pattern
    self.sql_ignore_pattern = /^(SHOW FIELDS |SET SQL_AUTO_IS_NULL|SET NAMES |EXPLAIN |BEGIN|COMMIT)/
    
    cattr_accessor :app_path_pattern
    self.app_path_pattern = Regexp.new(Regexp.quote("/app/"))
    
    cattr_accessor :trim_root_path
    self.trim_root_path = RAILS_ROOT[-1] == "/" ? RAILS_ROOT : RAILS_ROOT + "/"
    
    cattr_accessor :storage_backend
    self.storage_backend = :json    # or :fastercsv
    
    cattr_accessor :profile_self
    self.profile_self = false

    attr_accessor :last_stats_flush
    attr_accessor :query_sites
    attr_accessor :profile_data_directory
    
    def self.instance
      Thread.current[:active_record_profiler_collector] ||= Collector.new
    end
    
    def self.profiler_enabled?
      profile_environments.include?(ENV['RAILS_ENV'])
    end
    
    def self.profile_self?
      self.profile_self
    end
    
    def self.clear_data
      dir = Dir.new(profile_dir)
      prof_files = dir.entries.select{ |filename| /.prof$/.match(filename) }.map{ |filename| File.join(dir.path, filename) }
      FileUtils.rm(prof_files) if prof_files.size > 0
    end

    def initialize
      @query_sites = {}
      @last_stats_flush = nil
      @profile_data_directory = self.class.profile_dir
    end
    
    def call_location_name
      find_app_call_location || 'Non-application code'
    end
    
    def record_caller_info(location, seconds, sql)
      return if sql_ignore_pattern.match(sql)
      
      update_counts(location, seconds, 1, sql)
    end
    
    def record_self_info(seconds, name)
      record_caller_info(trim_location(caller.first), seconds, name)
    end
    
    def should_flush_stats?
      self.last_stats_flush ||= Time.now
      return(Time.now > self.last_stats_flush + stats_flush_period)
    end
    
    def flush_query_sites_statistics
      pid = $$
      thread_id = Thread.current.object_id
      flush_time = Time.now
      site_count = self.query_sites.keys.size
      RAILS_DEFAULT_LOGGER.info("Flushing ActiveRecordProfiler statistics for PID #{pid} THR #{thread_id} at #{flush_time} (#{site_count} sites).")
      
      if (site_count > 0)
        FileUtils.makedirs(self.profile_data_directory)
      
        filename = File.join(self.profile_data_directory, "#{flush_time.strftime(DATETIME_FORMAT)}.#{pid}-#{thread_id}.prof")
        write_file(filename)
        
{'fraction_non_alphanumeric': 0.08464867617107943, 'fraction_numerical': 0.004837067209775967, 'mean_word_length': 3.163751987281399, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 11, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4527604', 'n_tokens_mistral': 2685, 'n_tokens_neox': 2634, 'n_words': 552}