<commit_before>require 'yajl'

module Tritium
  module Engines
    require_relative '../base'
    class Debug < Engines::Base
      require_relative 'database'
      require_relative 'step'
      attr :root_step
      
      def initialize(script_string, options = {})
        super
        
        if ENV["TEST"].nil?
          @tmp_dir = File.join(@script_path, "../tmp")
          Dir.mkdir(@tmp_dir) unless File.directory?(@tmp_dir)
          
          # Dump the compiled script to the /tmp folder as script.ts
          script_file = File.join(@tmp_dir, "script.ts")
          File.open(script_file, "w+") { |f| f.write(@root_instruction.to_script) }
        end
        if options[:tr_dbg_to_db]
          @tr_dbg_to_db = true
        else
          @tr_dbg_to_db = false
        end
      end

      def run(doc, options = {})
        env = options["env"] || options[:env] || {}
        
        # Set this so that we run a full debug stack on every execution
        env["debug"] = "main"
        
        start = Time.now

        # Actually run the code
        @root_step = Step::Text.new(@root_instruction)
        @root_step.logger = @logger
        global_debug = {}
        export_vars = []
        @root_step.execute(doc.dup, env, global_debug, export_vars)
        
        took = Time.now - start
        unless ENV["TEST"]
          @logger.stats("Script took #{took} sec to process") if @logger.respond_to? :stats
        end

        #return [@root_step.object, export_vars] if ENV["TEST"]

        # If we called debug(), then do all of this
        
        print_stats(@root_step)
  
        if @tr_dbg_to_db
          debug_file = File.join(@tmp_dir, "debug.sqlite")
        
          @logger.info "DEBUG_TO_DB START! (to #{debug_file})"
        
          ## Build the new DB object
          db = Database.new(debug_file)

          ## Insert everything into the DB
          db.insert_instruction(@root_instruction)
          db.process_debug(global_debug)
        
          @logger.info "DEBUG_TO_DB FINISHED!"
        end
        
        ## Return the result object
        [@root_step.object, export_vars]
      end

      def print_stats(root_step)
        selectors = []

        root_step.each do |step|
          if step.debug[:search_time]
            selectors << step
          end
        end
        
        selectors = selectors.sort_by { |s| s.debug[:search_time] }
        selectors.reverse!
        @logger.info "Slowest Searches:"

        selectors[0..8].each_with_index do |step, index|
          @logger.info "##{index + 1} #{(step.debug[:search_time] * 1000).to_i}ms - #{step.instruction.args.first.inspect} in #{step.instruction.location}"
        end
      end

      <commit_msg>insert instructions to DB when it is created; save all execution steps of all instructions rather than overwrite the previous<commit_after>require 'yajl'

module Tritium
  module Engines
    require_relative '../base'
    class Debug < Engines::Base
      require_relative 'database'
      require_relative 'step'
      attr :root_step
      
      def initialize(script_string, options = {})
        super
        
        if ENV["TEST"].nil?
          @tmp_dir = File.join(@script_path, "../tmp")
          Dir.mkdir(@tmp_dir) unless File.directory?(@tmp_dir)
          
          # Dump the compiled script to the /tmp folder as script.ts
          script_file = File.join(@tmp_dir, "script.ts")
          File.open(script_file, "w+") { |f| f.write(@root_instruction.to_script) }
        end
        if options[:tr_dbg_to_db]
          debug_file = File.join(@tmp_dir, "debug.sqlite")
          @db = Database.new(debug_file, @root_instruction)
        else
          @db = nil
        end
      end

      def run(doc, options = {})
        env = options["env"] || options[:env] || {}
        
        # Set this so that we run a full debug stack on every execution
        env["debug"] = "main"
        
        start = Time.now

        # Actually run the code
        @root_step = Step::Text.new(@root_instruction)
        @root_step.logger = @logger
        global_debug = {}
        export_vars = []
        @root_step.execute(doc.dup, env, global_debug, export_vars)
        
        took = Time.now - start
        unless ENV["TEST"]
          @logger.stats("Script took #{took} sec to process") if @logger.respond_to? :stats
        end

        #return [@root_step.object, export_vars] if ENV["TEST"]

        # If we called debug(), then do all of this
        
        print_stats(@root_step)
        
        #dump steps into DB if needed 
        @db.process_debug(global_debug) unless @db.nil? 

        ## Return the result object
        [@root_step.object, export_vars]
      end

      def print_stats(root_step)
        selectors = []

        root_step.each do |step|
          if step.debug[:search_time]
            selectors << step
          end
        end
        
        selectors = selectors.sort_by { |s| s.debug[:search_time] }
        selectors.reverse!
        @logger.info "Slowest Searches:"

        selectors[0..8].each_with_index do |step, index|
          @logger.info "##{index + 1} #{(step.debug[:search_time] * 1000).to_i}ms - #{step.instruction.args.first.inspect} in #{step.instruction.location}"
        end
      end

      
{'fraction_non_alphanumeric': 0.109375, 'fraction_numerical': 0.0026355421686746986, 'mean_word_length': 1.9353591160220995, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6245423', 'n_tokens_mistral': 1615, 'n_tokens_neox': 1561, 'n_words': 388}