<commit_before>require 'stackprof/report'
require 'rbtrace/rbtracer'

module StackProf
  module Remote
    # ProcessReportCollector handles the work of actually starting,
    # stopping, and collecting the dumps from the StackProf profiler.
    #
    # Internally it uses RBTrace to execute the start/stop methods
    # against all runnign processes that match pids found by the :pid_finder
    # option. By default this matches unicorn workers.
    class ProcessReportCollector
      DEFAULT_OPTIONS = {
        :pid_finder => -> {
          `pgrep -f 'unicorn worker'`.strip.split.collect {|p| p.to_i }
        },
        :mode => :cpu,
        :interval => 1000,
        :raw => true,
        :path => 'tmp'
      }.freeze

      def initialize(options = {})
        @options = DEFAULT_OPTIONS.merge(options)
        collect_pids
      end

      def logger
        StackProf::Remote::Middleware.logger
      end

      def start
        command = "StackProf.start(mode: #{@options[:mode].inspect}, interval: #{@options[:interval].inspect}, raw: #{@options[:raw].inspect})"
        execute(command)
      end

      def stop
        command = "StackProf.stop"
        execute(command)
      end

      def save
        command = "StackProf::Remote::ReportSaver.save('#{@options[:path]}')"
        @saved_files = execute(command)
      end

      def marshaled_results
        if @saved_files
          logger.debug "[stackprof] Saved Files #{@saved_files.inspect}"
          saved_data = @saved_files.collect {|f|
            f = f.gsub(/"/,'') # RBTrace returns double quoted strings
            if File.readable?(f)
               Marshal.load(File.read(f))
            else
              logger.error "[stackprof] File #{f} not readable by process #{Process.pid}"
            end
          }.compact
          Marshal.dump(saved_data)
        end
      end

      def self.report_from_marshaled_results(marshaled_data)
        data = Marshal.load(marshaled_data)
        if data.is_a?(Array)
          puts "Loading results from #{data.length} workers"
          data.inject(nil) {|sum, d| (sum && d) ? StackProf::Report.new(d) + sum : StackProf::Report.new(d) }
        else
          StackProf::Report.new(d)
        end
      end

      private
      def collect_pids
        logger.debug "[stackprof] Collecting PIDs"
        @pids = @options[:pid_finder].call
        @pids -= [Process.pid]
        logger.debug "[stackprof] Found PIDs #{@pids.inspect} and current #{Process.pid}"<commit_msg>Switch to using compact<commit_after>require 'stackprof/report'
require 'rbtrace/rbtracer'

module StackProf
  module Remote
    # ProcessReportCollector handles the work of actually starting,
    # stopping, and collecting the dumps from the StackProf profiler.
    #
    # Internally it uses RBTrace to execute the start/stop methods
    # against all runnign processes that match pids found by the :pid_finder
    # option. By default this matches unicorn workers.
    class ProcessReportCollector
      DEFAULT_OPTIONS = {
        :pid_finder => -> {
          `pgrep -f 'unicorn worker'`.strip.split.collect {|p| p.to_i }
        },
        :mode => :cpu,
        :interval => 1000,
        :raw => true,
        :path => 'tmp'
      }.freeze

      def initialize(options = {})
        @options = DEFAULT_OPTIONS.merge(options)
        collect_pids
      end

      def logger
        StackProf::Remote::Middleware.logger
      end

      def start
        command = "StackProf.start(mode: #{@options[:mode].inspect}, interval: #{@options[:interval].inspect}, raw: #{@options[:raw].inspect})"
        execute(command)
      end

      def stop
        command = "StackProf.stop"
        execute(command)
      end

      def save
        command = "StackProf::Remote::ReportSaver.save('#{@options[:path]}')"
        @saved_files = execute(command)
      end

      def marshaled_results
        if @saved_files
          logger.debug "[stackprof] Saved Files #{@saved_files.inspect}"
          saved_data = @saved_files.collect {|f|
            f = f.gsub(/"/,'') # RBTrace returns double quoted strings
            if File.readable?(f)
               Marshal.load(File.read(f))
            else
              logger.error "[stackprof] File #{f} not readable by process #{Process.pid}"
            end
          }.compact
          Marshal.dump(saved_data)
        end
      end

      def self.report_from_marshaled_results(marshaled_data)
        data = Marshal.load(marshaled_data)
        if data.is_a?(Array)
          puts "Loading results from #{data.length} workers"
          data.compact.inject(nil) do |sum, d|
            sum ? StackProf::Report.new(d) + sum : StackProf::Report.new(d)
          end
        else
          StackProf::Report.new(d)
        end
      end

      private
      def collect_pids
        logger.debug "[stackprof] Collecting PIDs"
        @pids = @options[:pid_finder].call
        @pids -= [Process.pid]
        logger.debug "[stackprof] Found PIDs #{@pids.inspect} and current #{Process.pid}"
{'fraction_non_alphanumeric': 0.11005353955978585, 'fraction_numerical': 0.001586357326987904, 'mean_word_length': 2.2168367346938775, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 15, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '927639', 'n_tokens_mistral': 1497, 'n_tokens_neox': 1445, 'n_words': 371}