<commit_before>module Ramaze::Helper
  module GetPendingChanges
    include XYZ
    def flat_list_pending_changes_in_datacenter(datacenter_id)
      last_level = pending_changes_one_level_raw(:datacenter,[datacenter_id])
      ret = Array.new
      while not last_level.empty?
        ret += last_level
        last_level = pending_changes_one_level_raw(:state_change,last_level.map{|x|x[:id]})
      end
      remove_dups_and_proc_related_components(ret)
    end


    def pending_changes_one_level_raw(parent_model_name,id_list)
      actions = 
        pending_create_node(parent_model_name,id_list) + 
        pending_install_component(parent_model_name,id_list) +
        pending_changed_attribute(parent_model_name,id_list)
    end

    def pending_create_node(parent_model_name,id_list)
      parent_field_name = XYZ::DB.parent_field(parent_model_name,:state_change)
      sp_hash = {
        :relation => :state_change,
        :filter => [:and,
                    [:oneof, parent_field_name,id_list],
                    [:eq, :type, "create_node"],
                    [:eq, :status, "pending"]],
        :columns => [:id,:relative_order,:type,:created_node,parent_field_name,:state_change_id].uniq
      }
      get_objects_from_sp_hash(sp_hash)
    end

    def pending_install_component(parent_model_name,id_list)
      parent_field_name = XYZ::DB.parent_field(parent_model_name,:state_change)
      sp_hash = {
        :relation => :state_change,
        :filter => [:and,
                    [:oneof, parent_field_name, id_list],
                    [:eq, :type, "install_component"],
                    [:eq, :status, "pending"]],
        :columns => [:id, :relative_order,:type,:installed_component,parent_field_name,:state_change_id].uniq
      }
      sc_with_direct_cmps = get_objects_from_sp_hash(sp_hash)
      add_related_components(sc_with_direct_cmps)
    end

    def pending_changed_attribute(parent_model_name,id_list)
      parent_field_name = XYZ::DB.parent_field(parent_model_name,:state_change)
      sp_hash = {
        :relation => :state_change,
        :filter => [:and,
                    [:oneof, parent_field_name, id_list],
                    [:eq, :type, "setting"],
                    [:eq, :status, "pending"]],
        :columns => [:id, :relative_order,:type,:changed_attribute,parent_field_name,:state_change_id].uniq
      }      
      sc_with_direct_cmps = get_objects_from_sp_hash(sp_hash)
      add_related_components(sc_with_direct_cmps)
    end

    def add_related_components(sc_with_direct_cmps)
      cols = [:id,:display_name,:basic_type,:external_ref,:node_node_id,:only_one_per_node,:extended_base_id]
      related_cmps = Component.get_components_related_by_mixins(sc_with_direct_cmps.map{|sc|sc[:component]},cols)
      #TODO: assumption that cmps only appear once in sc_with_direct_cmps
      component_index = Hash.new
      sc_with_direct_cmps.each do |sc|
        cmp_id = sc[:component][:id]
        unless component_index[cmp_id]
          component_index[cmp_id] = sc
        else
          Log.error("component id #{cmp_id.to_s} appears more than once in sc_with_direct_cmps")
        end
      end
      sc_with_related_cmps = Array.new
      related_cmps.map do |cmp|
        cmp[:assoc_component_ids].each do |cmp_id|
          related_sc = component_index[cmp_id].merge(:component => cmp)
          sc_with_related_cmps << related_sc
        end
      end

      sc_with_direct_cmps + sc_with_related_cmps
    end

    def remove_dups_and_proc_related_components(state_changes)
      indexed_ret = Hash.new
      #remove duplicates wrt component and process linked_ids
      state_changes.each do |sc|
        if sc[:type] == "create_node"
          indexed_ret[sc[:node][:id]] = augment_with_linked_id(sc,sc[:id])
        elsif ["setting","install_component"].include?(sc[:type])
          indexed_ret[sc[:component][:id]] = augment_with_linked_id(indexed_ret[sc[:component][:id]] || sc.reject{|k,v|[:attribute].include?(k)},sc[:id])
        else
          Log.error("unexpected type #{sc[:type]}; ignoring")
        end
      end
      indexed_ret.values
    end

   private
    def augment_with_linked_id(state_change,id)<commit_msg>working on pulling related components for commit<commit_after>module Ramaze::Helper
  module GetPendingChanges
    include XYZ
    def flat_list_pending_changes_in_datacenter(datacenter_id)
      last_level = pending_changes_one_level_raw(:datacenter,[datacenter_id])
      ret = Array.new
      while not last_level.empty?
        ret += last_level
        last_level = pending_changes_one_level_raw(:state_change,last_level.map{|x|x[:id]})
      end
      remove_dups_and_proc_related_components(ret)
    end


    def pending_changes_one_level_raw(parent_model_name,id_list)
      actions = 
        pending_create_node(parent_model_name,id_list) + 
        pending_install_component(parent_model_name,id_list) +
        pending_changed_attribute(parent_model_name,id_list)
    end

    def pending_create_node(parent_model_name,id_list)
      parent_field_name = XYZ::DB.parent_field(parent_model_name,:state_change)
      sp_hash = {
        :relation => :state_change,
        :filter => [:and,
                    [:oneof, parent_field_name,id_list],
                    [:eq, :type, "create_node"],
                    [:eq, :status, "pending"]],
        :columns => [:id,:relative_order,:type,:created_node,parent_field_name,:state_change_id].uniq
      }
      get_objects_from_sp_hash(sp_hash)
    end

    def pending_install_component(parent_model_name,id_list)
      parent_field_name = XYZ::DB.parent_field(parent_model_name,:state_change)
      sp_hash = {
        :relation => :state_change,
        :filter => [:and,
                    [:oneof, parent_field_name, id_list],
                    [:eq, :type, "install_component"],
                    [:eq, :status, "pending"]],
        :columns => [:id, :relative_order,:type,:installed_component,parent_field_name,:state_change_id].uniq
      }
      sc_with_direct_cmps = get_objects_from_sp_hash(sp_hash)
      add_related_components(sc_with_direct_cmps)
    end

    def pending_changed_attribute(parent_model_name,id_list)
      parent_field_name = XYZ::DB.parent_field(parent_model_name,:state_change)
      sp_hash = {
        :relation => :state_change,
        :filter => [:and,
                    [:oneof, parent_field_name, id_list],
                    [:eq, :type, "setting"],
                    [:eq, :status, "pending"]],
        :columns => [:id, :relative_order,:type,:changed_attribute,parent_field_name,:state_change_id].uniq
      }      
      sc_with_direct_cmps = get_objects_from_sp_hash(sp_hash)
      add_related_components(sc_with_direct_cmps)
    end

    def add_related_components(sc_with_direct_cmps)
      component_index = Hash.new
      sc_with_direct_cmps.each do |sc|
        cmp_id = sc[:component][:id]
        unless component_index[cmp_id]
          component_index[cmp_id] = sc
        else
          if sc[:type] == "install_component" 
            component_index[cmp_id] = sc
          end
        end
      end

      cols = [:id,:display_name,:basic_type,:external_ref,:node_node_id,:only_one_per_node,:extended_base_id]
      cmps_in_sc = component_index.values.map{|sc|sc[:component]}
      related_cmps = Component.get_components_related_by_mixins(cmps_in_sc,cols)
      #TODO: assumption that cmps only appear once in sc_with_direct_cmps

      sc_with_related_cmps = Array.new
      related_cmps.map do |cmp|
        cmp[:assoc_component_ids].each do |cmp_id|
          related_sc = component_index[cmp_id].merge(:component => cmp)
          sc_with_related_cmps << related_sc
        end
      end

      sc_with_direct_cmps + sc_with_related_cmps
    end

    def remove_dups_and_proc_related_components(state_changes)
      indexed_ret = Hash.new
      #remove duplicates wrt component and process linked_ids
      state_changes.each do |sc|
        if sc[:type] == "create_node"
          indexed_ret[sc[:node][:id]] = augment_with_linked_id(sc,sc[:id])
          #TODO: ordering may do thsi anyway, but do we explicitly want to make sure if both setting adn isnatll use install as type
        elsif ["setting","install_component"].include?(sc[:type])
          indexed_ret[sc[:component][:id]] = augment_with_linked_id(indexed_ret[sc[:component][:id]] || sc.reject{|k,v|[:attribute].include?(k)},sc[:id])
        else
          Log.error("unexpected type #{sc[:type]}; ignoring")
        end
      end
      indexed_ret.values
    end

   private
    def augment_with_linked_id(state_change,id)
{'fraction_non_alphanumeric': 0.10261755848969191, 'fraction_numerical': 0.0, 'mean_word_length': 3.1674710424710426, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 23, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4216625', 'n_tokens_mistral': 2876, 'n_tokens_neox': 2787, 'n_words': 411}