<commit_before>require "experiment_progress_bar"
require "set"

class Experiment < ActiveRecord::Base
  belongs_to :user
  has_many :experiment_instances
  has_one :experiment_progress_bar
  
  attr_accessor :parametrization_info, :progress_bar, :data_farming_experiment

  PROGRESS_BAR_THRESHOLD = 10000

  def self.find_in_db(experiment_id)
    experiment_id = experiment_id.to_i
    experiment_hash = ExperimentInstanceDb.default_instance.get_experiment_info(experiment_id)
    
    if not experiment_hash.nil?
      additional_info = {
        "labels" => experiment_hash["labels"], 
        "value_list" => experiment_hash["value_list"], 
        "multiply_list" => experiment_hash["multiply_list"]
      }
      
      experiment_hash.delete("experiment_id")
      experiment_hash.delete("_id")
      experiment_hash.delete("labels")
      experiment_hash.delete("value_list")
      experiment_hash.delete("multiply_list")
      
      e = Experiment.new(experiment_hash)
      e.id = experiment_id
      e.parametrization_info = additional_info
      e.experiment_progress_bar = ExperimentProgressBar.new({:experiment_id => e.id})

      e.data_farming_experiment = DataFarmingExperiment.find_by_experiment_id(experiment_id)
      
      return e
    end
    
    nil
  end

  def experiment_size
    if not data_farming_experiment.nil?
      data_farming_experiment.experiment_size
    else
      super
    end
  end
=begin
  
  # cache related buggy functions 
  
  def self.find(id)
    cached = Rails.cache.read("experiment_#{id}")
    
    if cached.nil?
      # Rails.logger.debug("Experiment #{id} NOT IN CACHE")
      # cached = Experiment.find_by_id(id, :include => [:experiment_progress_bar])
      cached = Experiment.find_by_id(id)
      Rails.cache.write("experiment_#{id}", cached, :expires_in => 600.seconds)
    else
      # Rails.logger.debug("Experiment #{id} IN CACHE")
    end
    
    cached
  end
  
  def experiment_progress_bar
    cache_key = "progress_bar_#{self.id}"
    cached = Rails.cache.read(cache_key)
    
    if cached.nil?
      # Rails.logger.debug("ProgressBar for experiment #{self.id} NOT IN CACHE")
      cached = ExperimentProgressBar.find_by_experiment_id(self.id)
      Rails.cache.write(cache_key, cached, :expires_in => 600.seconds)
    else
      # Rails.logger.debug("ProgressBar for experiment #{self.id} IN CACHE")
    end
    
    cached
  end
  
=end
  
  def save_and_cache
    #Rails.cache.write("experiment_#{id}", self, :expires_in => 600.seconds)
    self.save 
  end

  def get_statistics
    all  = ExperimentInstance.count_with_query(self.id)
    done = ExperimentInstance.count_with_query(self.id, {"is_done" => true})
    sent = ExperimentInstance.count_with_query(self.id, {"to_sent" => false, "is_done" => false})

    return all, done, sent
  end

  def argument_names
    first_instance = ExperimentInstance.find_by_id(self.id, 1)

    first_instance.arguments.split(",").map{|arg| ParameterForm.parameter_uid_for_r(arg)}.join(",")
  end

  def range_arguments
    self.parametrization.split(",").select{|x| x.split("=").last == "range"}.
                                    map{|x| ParameterForm.parameter_uid_for_r(x.split("=").first)}
  end

  def parametrization_of(parameter_r_id)
    self.parametrization.split(",").each do |param_parametrization|
      parameter_uid, parametrization_type = param_parametrization.split("=")

      return parameter_uid, parametrization_type if ParameterForm.parameter_uid_for_r(parameter_uid) == parameter_r_id
    end

    nil
  end

  def generated_parameter_values_for(parameter_uid)
    instance = ExperimentInstance.find_by_id(self.id, 1)
    
    Rails.logger.debug("Parameter UID: #{parameter_uid}")
    Rails.logger.debug("instance.arguments: #{instance.arguments.split(",").map{|x| ParameterForm.parameter_uid_for_r(x)}}")
    
    param_index = instance.arguments.split(",").map{|x| ParameterForm.parameter_uid_for_r(x)}.index(parameter_uid)
    param_value = instance.values.split(",")[param_index]

    find_exp = "^"
    find_exp += "(\\d+\\.\\d+,){#{param_index}}" if param_index > 0
    find_exp = /#{find_exp}#{param_value}/

    query_hash = { "values" => { "$not" => find_exp }}
    option_hash = {:fields => ["values"]}

    param_values = ExperimentInstance.raw_find_by_query(self.id, query_hash, option_hash).
        map{|x| x["values"].split(",")[param_index]}.uniq + [param_value]

    param_values.map{|x| x.to_f}.sort
  end

  def self.running_experiments
    Experiment.where(:is_running => true).all(:order => "start_at DESC").to_a
  end

  def file_with_ids_path
    File.join(Rails.root, "tmp", "manager_#{Rails.configuration.manager_id}_exp_#{self.id}.dat")
  end

  def get_next_instance
    create_file_with_ids if not File.exist?(file_with_ids_path)

    begin
      instance_to_check = self.fetch_instance_from_db

      #if instance_to_check
        # instance_to_check.to_sent = false
        # instance_to_check.sent_at = Time.now
        # instance_to_check.save
      #end

      return instance_to_check

    rescue Exception => e
      logger.debug("get_next_instance --- #{e}")
    end

    nil
  end

  def fetch_instance_from_db
    begin
      self.send(self.scheduling_policy + "_scheduling")
    rescue Exception => e
      logger.debug("get_next_instance --- #{e}")
      nil
    end
  end

  def experiment_file_path
    File.join(self.data_folder_path, self.experiment_file)
  end

  def data_folder_path
    if self.experiment_file != nil then
      File.join(Rails.configuration.eusas_data_path, self.experiment_file.split('.')[0])
    else
      nil
    end
  end

  def self.experiments_info(conditions)
    ids = ActiveRecord::Base.connection.select_values("SELECT id FROM experiments WHERE #{conditions}")

    return [], [], [] if ids.empty?
    # TODO FIMXE should work but in the background
    dones = []
    ids.each do |experiment_id|
      dones << 0
      #dones << ExperimentInstance.count_with_query(experiment_id, { "is_done" => true })
    end

    sql = "SELECT id, experiment_size, vm_counter, start_at, experiment_name, end_at " +
        "FROM experiments WHERE id IN (#{ids.join(",")})"
    experiment_info = {}
    ActiveRecord::Base.connection.execute(sql).each do |row|
      experiment_info[row[0]] = row[1..5]
    end

    return ids, dones, experiment_info
  end

  def create_progress_bar
    progress_bar = ExperimentProgressBar.create(:experiment_id => self.id)
    progress_bar.insert_initial_bar(self.experiment_size)
    
    progress_bar.save
  end

  def result_file
    File.join(self.data_folder_path, "results.csv")
  end
  
  def result_file_for_moe(moe_name)
    File.join(self.data_folder_path, "results-#{moe_name}.csv")
  end

  def moe_names
    moe_name_set = Set.new
    
    ExperimentInstance.raw_find_by_query(self.id, { "is_done" => true }, { :fields => ["result"], :limit => (self.experiment_size / 2) }).each do |instance_doc|
      instance_doc["result"].split(",").each{ |x| moe_name_set.add(x.split("=")[0]) }
    end
    
    moe_name_set.empty? ? nil : moe_name_set.to_a
  end

  def result_names
    moe_name_set = Set.new
    result_limit = self.experiment_size < 5000 ? self.experiment_size : (self.experiment_size / 2)

    ExperimentInstance.raw_find_by_query(self.id, { is_done: true }, { fields: 'result', limit: result_limit }).each do |simulation_doc|
      moe_name_set += simulation_doc['result'].keys
    end

    moe_name_set.empty? ? nil : moe_name_set.to_a
  end

  def create_result_file
    File.delete(self.result_file) if File.exist?(self.result_file)

    File.open(self.result_file, "w+") do |f|
      # writing arguments' and moes' names from the first done instance
      moes = ExperimentInstance.get_first_done(self.id).result.split(",").map { |i| i.split("=")[0] }.join(",")
      f.puts("#{argument_names},#{moes}")

      # getting every completed instance and writes input parameters' values and results' values
      ExperimentInstance.raw_find_by_query(self.id, { "is_done" => true }, {:fields => ["values","result"]}).each do |instance_doc|
        # Rails.logger.debug { "Instance Id: #{instance_doc["id"]} --- Input size: #{instance_doc["values"].split(",").size} --- Results size: #{instance_doc["result"].split(',').map { |item| item.split("=")[1] }.size}" }
        f.puts("#{instance_doc["values"]},#{instance_doc["result"].split(',').map { |item| item.split("=")[1] }.join(",")}")
      end
    end

  end
  
  def create_result_file_for(moe_name)
    file_name = result_file_for_moe(moe_name)
    File.delete(file_name) if File.exist?(file_name)

    File.open(file_name, "w+") do |f|
      # writing arguments' and moe names from the first done instance
      f.puts("#{argument_names},#{moe_name}")

      # getting every completed instance and writes input parameters' values and results' values
      
      ExperimentInstance.raw_find_by_query(self.id, { "is_done" => true }, {:fields => ["values","result"]}).each do |instance_doc|
        moe_name_index = instance_doc["result"].index("#{moe_name}")
        next if moe_name_index.nil?
        
        last_index = (instance_doc["result"].index(",", moe_name_index)).nil? ? instance_doc["result"].size : instance_doc["result"].index(",", moe_name_index) 
        moe_value = instance_doc["result"][(moe_name_index+moe_name.size+1)...last_index]
        f.puts("#{instance_doc["values"]},#{moe_value}")
      end
    end
    
    file_name
  end
  
  def create_result_file_for_scatter_plot(x_axis, y_axis)
    file_name = File.join(self.data_folder_path, "results-scatter-#{x_axis}-#{y_axis}.csv")
    
    File.open(file_name, "w+") do |f|
      f.puts("#{x_axis},#{y_axis}")
      
      x_axis_index = (argument_names.include?(x_axis+",") or argument_names.include?(","+x_axis)) ? argument_names.split(",").index(x_axis) : nil
      y_axis_index = (argument_names.include?(y_axis+",") or argument_names.include?(","+y_axis)) ? argument_names.split(",").index(y_axis) : nil   
      
      ExperimentInstance.raw_find_by_query(self.id, { "is_done" => true }, {:fields => ["values","result"]}).each do |instance_doc|
        x_axis_value = if x_axis_index 
            instance_doc["values"].split(",")[x_axis_index]
          else
            moe_name_index = instance_doc["result"].index("#{x_axis}")
            next if moe_name_index.nil?
            
            last_index = (instance_doc["result"].index(",", moe_name_index)).nil? ? instance_doc["result"].size : instance_doc["result"].index(",", moe_name_index) 
            instance_doc["result"][(moe_name_index+x_axis.size+1)...last_index]
          end
        
        y_axis_value = if y_axis_index 
            instance_doc["values"].split(",")[y_axis_index]
          else
            moe_name_index = instance_doc["result"].index("#{y_axis}")
            next if moe_name_index.nil?
            
            last_index = (instance_doc["result"].index(",", moe_name_index)).nil? ? instance_doc["result"].size : instance_doc["result"].index(",", moe_name_index) 
            instance_doc["result"][(moe_name_index+y_axis.size+1)...last_index]
          end      
        
        f.puts("#{x_axis_value},#{y_axis_value}")
      end
    end
    
    file_name
  end

  def completed_simulations_count_for(secs)
    query = { "is_done" => true, "done_at" => { "$gte" => (Time.now - secs)} }

    ExperimentInstance.count_with_query(self.id, query)
  end
  
  def make_simulation_logs_dir
    begin
      Dir::mkdir(self.data_folder_path) if not File.exists?(self.data_folder_path)
    rescue Exception => e
      logger.debug("Error: make_simulation_logs_dir - #{e.inspect}")
    end
  end
  
  def create_parameters_and_doe_groups(params_tab = nil)
     if not params_tab.nil?
       params_to_override, params_groups_for_doe = params_tab 
     else
       params_to_override = self.parameters.split("|").map{ |x| x.split("=") }
       params_groups_for_doe = self.doe_groups.split("|").map{ |x| x.split("=") }
     end

     begin
      parameters = DataFarmingScenario.get_and_override_parameters(self, params_to_override)
      #Rails.logger.debug("\nparameters = #{parameters}\n")
      doe_groups = DataFarmingScenario.create_groups_for_doe(params_groups_for_doe, parameters)
      #Rails.logger.debug("\ndoe_groups = #{doe_groups}\n")
     rescue Exception => e
       Rails.logger.debug("Error while creating parameters and doe groups --- #{e}")
       return [], []
     end

     return parameters, doe_groups
  end
  
  def generate_instance_configurations(start_id = 0)
    ExperimentInstanceDb.create_table_for(self.id)
    parameters, doe_groups = self.create_parameters_and_doe_groups

    return [], [] if parameters.size + doe_groups.size == 0

    counter, index_list, labels, value_list = prepare_factors_for_instance_generation(parameters, doe_groups)
    #Rails.logger.debug("=========== prepare_factors_for_instance_generation")
    #Rails.logger.debug("= index_list - #{index_list.inspect}")

    #value_list.each do |list|
    #  Rails.logger.debug(list.inspect)
    #end

    Rails.logger.debug("= labels.size - #{labels.size}")
    Rails.logger.debug("= value_list.size - #{value_list.size}")

    a = Time.now
    multiply_list = Array.new(value_list.size)
    multiply_list[-1] = 1
    (multiply_list.size - 2).downto(0) do |index|
      multiply_list[index] = multiply_list[index + 1] * value_list[index + 1].size
    end
    Rails.logger.debug("Time of preparing multiply_list: #{Time.now - a}")
    
    ExperimentInstanceDb.default_instance.store_experiment_info(self, labels, value_list, multiply_list)
  end

  def progress_bar_update(simulation_id, update_type)
    return if self.experiment_size < PROGRESS_BAR_THRESHOLD and update_type == "sent"

    parts_per_slot = parts_per_progress_bar_slot
    bar_index = ((simulation_id - 1) / parts_per_slot).floor

    increment_value = if update_type == "done"
                        (self.experiment_size < PROGRESS_BAR_THRESHOLD) ? 1 : 2
                      elsif update_type == "sent"
                        1
                      elsif update_type == "rollback"
                        -1
                      end

    begin
      progress_bar_table.update({:bar_num => bar_index}, "$inc" => {:bar_state => increment_value})
    rescue Exception => e
      Rails.logger.debug("Error in fastest update --- #{e}")
    end
  end

  # ======================= PRIVATE METHODS =============================
  private

  # scheduling policy methods: monte_carlo, sequential_forward, sequential_backward
  def monte_carlo_scheduling
    db = ExperimentInstanceDb.default_instance<commit_msg>Fixing problem with progress bar update.<commit_after>require "experiment_progress_bar"
require "set"

class Experiment < ActiveRecord::Base
  belongs_to :user
  has_many :experiment_instances
  has_one :experiment_progress_bar
  
  attr_accessor :parametrization_info, :progress_bar, :data_farming_experiment

  PROGRESS_BAR_THRESHOLD = 10000

  def self.find_in_db(experiment_id)
    experiment_id = experiment_id.to_i
    experiment_hash = ExperimentInstanceDb.default_instance.get_experiment_info(experiment_id)
    
    if not experiment_hash.nil?
      additional_info = {
        "labels" => experiment_hash["labels"], 
        "value_list" => experiment_hash["value_list"], 
        "multiply_list" => experiment_hash["multiply_list"]
      }
      
      experiment_hash.delete("experiment_id")
      experiment_hash.delete("_id")
      experiment_hash.delete("labels")
      experiment_hash.delete("value_list")
      experiment_hash.delete("multiply_list")
      
      e = Experiment.new(experiment_hash)
      e.id = experiment_id
      e.parametrization_info = additional_info
      e.experiment_progress_bar = ExperimentProgressBar.new({:experiment_id => e.id})

      e.data_farming_experiment = DataFarmingExperiment.find_by_experiment_id(experiment_id)
      
      return e
    end
    
    nil
  end

  def experiment_size
    if not data_farming_experiment.nil?
      data_farming_experiment.experiment_size
    else
      super
    end
  end
=begin
  
  # cache related buggy functions 
  
  def self.find(id)
    cached = Rails.cache.read("experiment_#{id}")
    
    if cached.nil?
      # Rails.logger.debug("Experiment #{id} NOT IN CACHE")
      # cached = Experiment.find_by_id(id, :include => [:experiment_progress_bar])
      cached = Experiment.find_by_id(id)
      Rails.cache.write("experiment_#{id}", cached, :expires_in => 600.seconds)
    else
      # Rails.logger.debug("Experiment #{id} IN CACHE")
    end
    
    cached
  end
  
  def experiment_progress_bar
    cache_key = "progress_bar_#{self.id}"
    cached = Rails.cache.read(cache_key)
    
    if cached.nil?
      # Rails.logger.debug("ProgressBar for experiment #{self.id} NOT IN CACHE")
      cached = ExperimentProgressBar.find_by_experiment_id(self.id)
      Rails.cache.write(cache_key, cached, :expires_in => 600.seconds)
    else
      # Rails.logger.debug("ProgressBar for experiment #{self.id} IN CACHE")
    end
    
    cached
  end
  
=end
  
  def save_and_cache
    #Rails.cache.write("experiment_#{id}", self, :expires_in => 600.seconds)
    self.save 
  end

  def get_statistics
    all  = ExperimentInstance.count_with_query(self.id)
    done = ExperimentInstance.count_with_query(self.id, {"is_done" => true})
    sent = ExperimentInstance.count_with_query(self.id, {"to_sent" => false, "is_done" => false})

    return all, done, sent
  end

  def argument_names
    first_instance = ExperimentInstance.find_by_id(self.id, 1)

    first_instance.arguments.split(",").map{|arg| ParameterForm.parameter_uid_for_r(arg)}.join(",")
  end

  def range_arguments
    self.parametrization.split(",").select{|x| x.split("=").last == "range"}.
                                    map{|x| ParameterForm.parameter_uid_for_r(x.split("=").first)}
  end

  def parametrization_of(parameter_r_id)
    self.parametrization.split(",").each do |param_parametrization|
      parameter_uid, parametrization_type = param_parametrization.split("=")

      return parameter_uid, parametrization_type if ParameterForm.parameter_uid_for_r(parameter_uid) == parameter_r_id
    end

    nil
  end

  def generated_parameter_values_for(parameter_uid)
    instance = ExperimentInstance.find_by_id(self.id, 1)
    
    Rails.logger.debug("Parameter UID: #{parameter_uid}")
    Rails.logger.debug("instance.arguments: #{instance.arguments.split(",").map{|x| ParameterForm.parameter_uid_for_r(x)}}")
    
    param_index = instance.arguments.split(",").map{|x| ParameterForm.parameter_uid_for_r(x)}.index(parameter_uid)
    param_value = instance.values.split(",")[param_index]

    find_exp = "^"
    find_exp += "(\\d+\\.\\d+,){#{param_index}}" if param_index > 0
    find_exp = /#{find_exp}#{param_value}/

    query_hash = { "values" => { "$not" => find_exp }}
    option_hash = {:fields => ["values"]}

    param_values = ExperimentInstance.raw_find_by_query(self.id, query_hash, option_hash).
        map{|x| x["values"].split(",")[param_index]}.uniq + [param_value]

    param_values.map{|x| x.to_f}.sort
  end

  def self.running_experiments
    Experiment.where(:is_running => true).all(:order => "start_at DESC").to_a
  end

  def file_with_ids_path
    File.join(Rails.root, "tmp", "manager_#{Rails.configuration.manager_id}_exp_#{self.id}.dat")
  end

  def get_next_instance
    create_file_with_ids if not File.exist?(file_with_ids_path)

    begin
      instance_to_check = self.fetch_instance_from_db

      #if instance_to_check
        # instance_to_check.to_sent = false
        # instance_to_check.sent_at = Time.now
        # instance_to_check.save
      #end

      return instance_to_check

    rescue Exception => e
      logger.debug("get_next_instance --- #{e}")
    end

    nil
  end

  def fetch_instance_from_db
    begin
      self.send(self.scheduling_policy + "_scheduling")
    rescue Exception => e
      logger.debug("get_next_instance --- #{e}")
      nil
    end
  end

  def experiment_file_path
    File.join(self.data_folder_path, self.experiment_file)
  end

  def data_folder_path
    if self.experiment_file != nil then
      File.join(Rails.configuration.eusas_data_path, self.experiment_file.split('.')[0])
    else
      nil
    end
  end

  def self.experiments_info(conditions)
    ids = ActiveRecord::Base.connection.select_values("SELECT id FROM experiments WHERE #{conditions}")

    return [], [], [] if ids.empty?
    # TODO FIMXE should work but in the background
    dones = []
    ids.each do |experiment_id|
      dones << 0
      #dones << ExperimentInstance.count_with_query(experiment_id, { "is_done" => true })
    end

    sql = "SELECT id, experiment_size, vm_counter, start_at, experiment_name, end_at " +
        "FROM experiments WHERE id IN (#{ids.join(",")})"
    experiment_info = {}
    ActiveRecord::Base.connection.execute(sql).each do |row|
      experiment_info[row[0]] = row[1..5]
    end

    return ids, dones, experiment_info
  end

  def create_progress_bar
    progress_bar = ExperimentProgressBar.create(:experiment_id => self.id)
    progress_bar.insert_initial_bar(self.experiment_size)
    
    progress_bar.save
  end

  def result_file
    File.join(self.data_folder_path, "results.csv")
  end
  
  def result_file_for_moe(moe_name)
    File.join(self.data_folder_path, "results-#{moe_name}.csv")
  end

  def moe_names
    moe_name_set = Set.new
    
    ExperimentInstance.raw_find_by_query(self.id, { "is_done" => true }, { :fields => ["result"], :limit => (self.experiment_size / 2) }).each do |instance_doc|
      instance_doc["result"].split(",").each{ |x| moe_name_set.add(x.split("=")[0]) }
    end
    
    moe_name_set.empty? ? nil : moe_name_set.to_a
  end

  def result_names
    moe_name_set = Set.new
    result_limit = self.experiment_size < 5000 ? self.experiment_size : (self.experiment_size / 2)

    ExperimentInstance.raw_find_by_query(self.id, { is_done: true }, { fields: 'result', limit: result_limit }).each do |simulation_doc|
      moe_name_set += simulation_doc['result'].keys
    end

    moe_name_set.empty? ? nil : moe_name_set.to_a
  end

  def create_result_file
    File.delete(self.result_file) if File.exist?(self.result_file)

    File.open(self.result_file, "w+") do |f|
      # writing arguments' and moes' names from the first done instance
      moes = ExperimentInstance.get_first_done(self.id).result.split(",").map { |i| i.split("=")[0] }.join(",")
      f.puts("#{argument_names},#{moes}")

      # getting every completed instance and writes input parameters' values and results' values
      ExperimentInstance.raw_find_by_query(self.id, { "is_done" => true }, {:fields => ["values","result"]}).each do |instance_doc|
        # Rails.logger.debug { "Instance Id: #{instance_doc["id"]} --- Input size: #{instance_doc["values"].split(",").size} --- Results size: #{instance_doc["result"].split(',').map { |item| item.split("=")[1] }.size}" }
        f.puts("#{instance_doc["values"]},#{instance_doc["result"].split(',').map { |item| item.split("=")[1] }.join(",")}")
      end
    end

  end
  
  def create_result_file_for(moe_name)
    file_name = result_file_for_moe(moe_name)
    File.delete(file_name) if File.exist?(file_name)

    File.open(file_name, "w+") do |f|
      # writing arguments' and moe names from the first done instance
      f.puts("#{argument_names},#{moe_name}")

      # getting every completed instance and writes input parameters' values and results' values
      
      ExperimentInstance.raw_find_by_query(self.id, { "is_done" => true }, {:fields => ["values","result"]}).each do |instance_doc|
        moe_name_index = instance_doc["result"].index("#{moe_name}")
        next if moe_name_index.nil?
        
        last_index = (instance_doc["result"].index(",", moe_name_index)).nil? ? instance_doc["result"].size : instance_doc["result"].index(",", moe_name_index) 
        moe_value = instance_doc["result"][(moe_name_index+moe_name.size+1)...last_index]
        f.puts("#{instance_doc["values"]},#{moe_value}")
      end
    end
    
    file_name
  end
  
  def create_result_file_for_scatter_plot(x_axis, y_axis)
    file_name = File.join(self.data_folder_path, "results-scatter-#{x_axis}-#{y_axis}.csv")
    
    File.open(file_name, "w+") do |f|
      f.puts("#{x_axis},#{y_axis}")
      
      x_axis_index = (argument_names.include?(x_axis+",") or argument_names.include?(","+x_axis)) ? argument_names.split(",").index(x_axis) : nil
      y_axis_index = (argument_names.include?(y_axis+",") or argument_names.include?(","+y_axis)) ? argument_names.split(",").index(y_axis) : nil   
      
      ExperimentInstance.raw_find_by_query(self.id, { "is_done" => true }, {:fields => ["values","result"]}).each do |instance_doc|
        x_axis_value = if x_axis_index 
            instance_doc["values"].split(",")[x_axis_index]
          else
            moe_name_index = instance_doc["result"].index("#{x_axis}")
            next if moe_name_index.nil?
            
            last_index = (instance_doc["result"].index(",", moe_name_index)).nil? ? instance_doc["result"].size : instance_doc["result"].index(",", moe_name_index) 
            instance_doc["result"][(moe_name_index+x_axis.size+1)...last_index]
          end
        
        y_axis_value = if y_axis_index 
            instance_doc["values"].split(",")[y_axis_index]
          else
            moe_name_index = instance_doc["result"].index("#{y_axis}")
            next if moe_name_index.nil?
            
            last_index = (instance_doc["result"].index(",", moe_name_index)).nil? ? instance_doc["result"].size : instance_doc["result"].index(",", moe_name_index) 
            instance_doc["result"][(moe_name_index+y_axis.size+1)...last_index]
          end      
        
        f.puts("#{x_axis_value},#{y_axis_value}")
      end
    end
    
    file_name
  end

  def completed_simulations_count_for(secs)
    query = { "is_done" => true, "done_at" => { "$gte" => (Time.now - secs)} }

    ExperimentInstance.count_with_query(self.id, query)
  end
  
  def make_simulation_logs_dir
    begin
      Dir::mkdir(self.data_folder_path) if not File.exists?(self.data_folder_path)
    rescue Exception => e
      logger.debug("Error: make_simulation_logs_dir - #{e.inspect}")
    end
  end
  
  def create_parameters_and_doe_groups(params_tab = nil)
     if not params_tab.nil?
       params_to_override, params_groups_for_doe = params_tab 
     else
       params_to_override = self.parameters.split("|").map{ |x| x.split("=") }
       params_groups_for_doe = self.doe_groups.split("|").map{ |x| x.split("=") }
     end

     begin
      parameters = DataFarmingScenario.get_and_override_parameters(self, params_to_override)
      #Rails.logger.debug("\nparameters = #{parameters}\n")
      doe_groups = DataFarmingScenario.create_groups_for_doe(params_groups_for_doe, parameters)
      #Rails.logger.debug("\ndoe_groups = #{doe_groups}\n")
     rescue Exception => e
       Rails.logger.debug("Error while creating parameters and doe groups --- #{e}")
       return [], []
     end

     return parameters, doe_groups
  end
  
  def generate_instance_configurations(start_id = 0)
    ExperimentInstanceDb.create_table_for(self.id)
    parameters, doe_groups = self.create_parameters_and_doe_groups

    return [], [] if parameters.size + doe_groups.size == 0

    counter, index_list, labels, value_list = prepare_factors_for_instance_generation(parameters, doe_groups)
    #Rails.logger.debug("=========== prepare_factors_for_instance_generation")
    #Rails.logger.debug("= index_list - #{index_list.inspect}")

    #value_list.each do |list|
    #  Rails.logger.debug(list.inspect)
    #end

    Rails.logger.debug("= labels.size - #{labels.size}")
    Rails.logger.debug("= value_list.size - #{value_list.size}")

    a = Time.now
    multiply_list = Array.new(value_list.size)
    multiply_list[-1] = 1
    (multiply_list.size - 2).downto(0) do |index|
      multiply_list[index] = multiply_list[index + 1] * value_list[index + 1].size
    end
    Rails.logger.debug("Time of preparing multiply_list: #{Time.now - a}")
    
    ExperimentInstanceDb.default_instance.store_experiment_info(self, labels, value_list, multiply_list)
  end

  def progress_bar_update(simulation_id, update_type)
    return if self.experiment_size > PROGRESS_BAR_THRESHOLD and update_type == 'sent'

    parts_per_slot = parts_per_progress_bar_slot
    bar_index = ((simulation_id - 1) / parts_per_slot).floor

    increment_value = if update_type == 'done'
                        (self.experiment_size > PROGRESS_BAR_THRESHOLD) ? 1 : 2
                      elsif update_type == 'sent'
                        1
                      elsif update_type == 'rollback'
                        -1
                      end

    begin
      progress_bar_table.update({:bar_num => bar_index}, "$inc" => {:bar_state => increment_value})
    rescue Exception => e
      Rails.logger.debug("Error in fastest update --- #{e}")
    end
  end

  # ======================= PRIVATE METHODS =============================
  private

  # scheduling policy methods: monte_carlo, sequential_forward, sequential_backward
  def monte_carlo_scheduling
    db = ExperimentInstanceDb.default_instance
{'fraction_non_alphanumeric': 0.13354915658544897, 'fraction_numerical': 0.0032714261373317432, 'mean_word_length': 3.441652792492811, 'pattern_counts': {'":': 0, '<': 17, '<?xml version=': 0, '>': 83, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4591591', 'n_tokens_mistral': 9937, 'n_tokens_neox': 9708, 'n_words': 1642}