<commit_before>module XYZ
  class TaskLog < Model
    def self.get_and_update_logs_content(task,assoc_nodes,log_filter)
      ret_info = assoc_nodes.inject({}){|h,n|h.merge(n[:task_id] => {:node => n})}
      log_type = :chef  #TODO: stub to just get chef run logs
      sp_hash = {
        :cols => [:id,:status,:type,:content, :task_id],
        :filter => [:and, [:oneof, :task_id, assoc_nodes.map{|n|n[:task_id]}],
                    [:eq, :type, log_type.to_s]]
      }
      task_log_mh = task.model_handle.createMH(:task_log)
      task_logs = Model.get_objects_from_sp_hash(task_log_mh,sp_hash)

      #associate logs with task_ids and kick of deferred job to get logs for all nodes that dont haev compleet logs in db
      task_logs.each do |task_log|
        task_id = task_log[:task_id]
        #implicit assumption that only one log per task
        ret_info[task_id] = {:log => task_log[:content],:status => task_log[:status]}
      end
      #TODO: compute status from parent task
      incl_assoc_nodes = ret_info.values.reject{|t|t[:status] == "complete"}.map{|info|info[:node]}
      task_pbuilderid_index = incl_assoc_nodes.inject({}){|h,n|h.merge(Node.pbuilderid(n) => n[:task_id])}
      callbacks = {
        :on_msg_received => proc do |msg|
          response = CommandAndControl.parse_response__get_logs(task,msg)
          if response[:status] == :ok
            task_id = task_pbuilderid_index[response[:pbuilderid]]
            task_idh = task.model_handle.createIDH(:id => task_id)
            TaskLog.create_or_update(task_idh,log_type.to_s,response[:log_content])
          else
            Log.error("error response for request to get log")
            #TODO: put some subset of this in error msg
            pp msg
          end
        end
      }
      CommandAndControl.request__get_logs(task,incl_assoc_nodes,callbacks,:log_type => log_type)
      ret_info.values.map{|x|x[:log]}
    end
    def self.create_or_update(task_idh,log_type,log_content)
      status = ParseLog.log_complete?(log_type,log_content) ? "complete" : "in_progress"
      #TODO: stub for testing
      status = "in_progress"

      #create if needed
      sp_hash = {
        :cols => [:id],
        :filter => [:eq, :type, log_type.to_s]
      }
      task_log_mh = task_idh.createMH(:task_log)
      existing = Model.get_objects_from_sp_hash(task_log_mh,sp_hash).first
      row = {
        :task_id => task_idh.get_id,
        :status => status,
        :type => log_type.to_s,
        :content => log_content
      }
      if existing
        id = existing[:id]
        ret = task_log_mh.createIDH(:id => id)
        row.merge!(:id => id)
        Model.update_from_rows(task_log_mh,[row])
      else
        row.merge!(:ref => log_type.to_s) 
        ret = Model.create_from_rows(task_log_mh,[row]).first
      end
      ret
    end
  end
end
<commit_msg>working on coordination between async get logs and db<commit_after>module XYZ
  class TaskLog < Model
    def self.get_and_update_logs_content(task,assoc_nodes,log_filter)
      ret_info = assoc_nodes.inject({}){|h,n|h.merge(n[:task_id] => {:node => n})}
      log_type = :chef  #TODO: stub to just get chef run logs
      sp_hash = {
        :cols => [:id,:status,:type,:content, :task_id],
        :filter => [:and, [:oneof, :task_id, assoc_nodes.map{|n|n[:task_id]}],
                    [:eq, :type, log_type.to_s]]
      }
      task_log_mh = task.model_handle.createMH(:task_log)
      task_logs = Model.get_objects_from_sp_hash(task_log_mh,sp_hash)

      #associate logs with task_ids and kick of deferred job to get logs for all nodes that dont haev compleet logs in db
      task_logs.each do |task_log|
        task_id = task_log[:task_id]
        #implicit assumption that only one log per task
        ret_info[task_id].merge!(:log => task_log[:content],:status => task_log[:status])
      end
      incl_assoc_nodes = ret_info.values.reject{|t|t[:status] == "complete"}.map{|info|info[:node]}
      task_pbuilderid_index = incl_assoc_nodes.inject({}){|h,n|h.merge(Node.pbuilderid(n) => n[:task_id])}
      callbacks = {
        :on_msg_received => proc do |msg|
          response = CommandAndControl.parse_response__get_logs(task,msg)
          if response[:status] == :ok
            task_id = task_pbuilderid_index[response[:pbuilderid]]
            task_idh = task.model_handle.createIDH(:id => task_id)
            TaskLog.create_or_update(task_idh,log_type.to_s,response[:log_content])
          else
            Log.error("error response for request to get log")
            #TODO: put some subset of this in error msg
            pp msg
          end
        end
      }
      CommandAndControl.request__get_logs(task,incl_assoc_nodes,callbacks,:log_type => log_type)
      ret_info.values.map{|x|x[:log]}
    end
    def self.create_or_update(task_idh,log_type,log_content)
      task_id = task_idh.get_id()
      status = ParseLog.log_complete?(log_type,log_content) ? "complete" : "in_progress"

      #create if needed
      sp_hash = {
        :cols => [:id],
        :filter => [:and, [:eq, :task_id, task_id],
                    [:eq, :type, log_type.to_s]]
      }
      task_log_mh = task_idh.createMH(:task_log)
      existing = Model.get_objects_from_sp_hash(task_log_mh,sp_hash).first
      row = {
        :task_id => task_id,
        :status => status,
        :type => log_type.to_s,
        :content => log_content
      }
      if existing
        id = existing[:id]
        ret = task_log_mh.createIDH(:id => id)
        row.merge!(:id => id)
        Model.update_from_rows(task_log_mh,[row])
      else
        row.merge!(:ref => log_type.to_s) 
        ret = Model.create_from_rows(task_log_mh,[row]).first
      end
      ret
    end
  end
end

{'fraction_non_alphanumeric': 0.1226546212647672, 'fraction_numerical': 0.0, 'mean_word_length': 2.7334630350194553, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 41, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5298076', 'n_tokens_mistral': 2017, 'n_tokens_neox': 1968, 'n_words': 371}