<commit_before>module XYZ
  class AttributeUpdateDerivedValues
    def self.update(attr_mh,update_deltas,opts={})
      ret = Array.new
      attr_ids = update_deltas.map{|r|r[:id]}
      critical_section(attr_ids) do
        ret = update_in_critical_section(attr_mh,update_deltas,opts={})
      end
      ret
    end

    def self.update_for_delete_links(attr_mh,links_info)
      ret = Array.new
      attr_ids = links_info.map{|l|l[:attribute_id]}
      critical_section(attr_ids) do
        ret = links_info.map{|link_info|update_attr_for_delete_link(attr_mh,link_info)}
      end
      ret
    end

   private
    Lock = Mutex.new
    def self.critical_section(attr_ids,&block)
      #passing in attr_ids, but not using now; may use if better to lock on per attribute basis
      Lock.synchronize{yield}
    end

    def self.update_in_critical_section(attr_mh,update_deltas,opts={})
      #break up by type of row and process and aggregate
      return Array.new if update_deltas.empty?
      ndx_update_deltas = update_deltas.inject({}) do |h,r|
        index = Aux::demodulize(r.class.to_s)
        (h[index] ||= Array.new) << r
        h
      end
      ndx_update_deltas.map do |type,rows|
        update_attribute_values_aux(type,attr_mh,rows,opts)
      end.flatten
    end

    def self.update_attribute_values_aux(type,attr_mh,update_deltas,opts={})
      case type
        when "OutputArrayAppend"
          update_attribute_values_array_append(attr_mh,update_deltas,opts)
        when "OutputPartial"
          update_attribute_values_partial(attr_mh,update_deltas,opts)
        else #TODO: below is for legacy form befor had Output objects; may convert to Output form
        raise Errow.new("Need to refactor to handle use of source_output_id")
        update_select_ds = SQL::ArrayDataset.create(db,update_deltas,attr_mh,:convert_for_update => true)
        Model.update_from_select(attr_mh,FieldSet.new(:attribute,[:value_derived]),update_select_ds,opts)
      end
    end

    #appends value to any array type; if the array does not exist already it creates it from fresh
    def self.update_attribute_values_array_append(attr_mh,array_slice_rows,opts={})
      ret = Array.new
      attr_link_updates = Array.new
      id_list = array_slice_rows.map{|r|r[:id]}
      Model.select_process_and_update(attr_mh,[:id,:value_derived],id_list) do |existing_vals|
        ndx_existing_vals = existing_vals.inject({}) do |h,r|
          h.merge(r[:id] => r[:value_derived])
        end
        attr_updates = array_slice_rows.map do |r|
          attr_id = r[:id]
          existing_val = ndx_existing_vals[attr_id]||[]
          offset = existing_val.size
          last_el = r[:array_slice].size-1
          index_map = r[:output_is_array] ?
          AttributeLink::IndexMap.generate_from_bounds(0,last_el,offset) :
            AttributeLink::IndexMap.generate_for_output_scalar(last_el,offset) 
          attr_link_update = {
            :id => r[:attr_link_id],
            :index_map => index_map<commit_msg>fising attribute propagation for simple case<commit_after>module XYZ
  class AttributeUpdateDerivedValues
    def self.update(attr_mh,update_deltas,opts={})
      ret = Array.new
      attr_ids = update_deltas.map{|r|r[:id]}
      critical_section(attr_ids) do
        ret = update_in_critical_section(attr_mh,update_deltas,opts={})
      end
      ret
    end

    def self.update_for_delete_links(attr_mh,links_info)
      ret = Array.new
      attr_ids = links_info.map{|l|l[:attribute_id]}
      critical_section(attr_ids) do
        ret = links_info.map{|link_info|update_attr_for_delete_link(attr_mh,link_info)}
      end
      ret
    end

   private
    Lock = Mutex.new
    def self.critical_section(attr_ids,&block)
      #passing in attr_ids, but not using now; may use if better to lock on per attribute basis
      Lock.synchronize{yield}
    end

    def self.update_in_critical_section(attr_mh,update_deltas,opts={})
      #break up by type of row and process and aggregate
      return Array.new if update_deltas.empty?
      ndx_update_deltas = update_deltas.inject({}) do |h,r|
        index = Aux::demodulize(r.class.to_s)
        (h[index] ||= Array.new) << r
        h
      end
      ndx_update_deltas.map do |type,rows|
        update_attribute_values_aux(type,attr_mh,rows,opts)
      end.flatten
    end

    def self.update_attribute_values_aux(type,attr_mh,update_deltas,opts={})
      case type
        when "OutputArrayAppend"
          update_attribute_values_array_append(attr_mh,update_deltas,opts)
        when "OutputPartial"
          update_attribute_values_partial(attr_mh,update_deltas,opts)
        else 
          update_attribute_values_simple(attr_mh,update_deltas,opts)
      end
    end


    def self.update_attribute_values_simple(attr_mh,update_hashes,opts={})
      ret = Array.new
      id_list = update_hashes.map{|r|r[:id]}
      Model.select_process_and_update(attr_mh,[:id,:value_derived],id_list) do |existing_vals|
        ndx_existing_vals = existing_vals.inject({}){|h,r|h.merge(r[:id] => r[:value_derived])}
        update_hashes.map do |r|
          attr_id = r[:id]
          existing_val = ndx_existing_vals[attr_id]
          replacement_row = {:id => attr_id, :value_derived => r[:value_derived]}
          ret << replacement_row.merge(:source_output_id => r[:source_output_id], :old_value_derived => existing_val)
          replacement_row
        end
      end
      ret
    end

    #appends value to any array type; if the array does not exist already it creates it from fresh
    def self.update_attribute_values_array_append(attr_mh,array_slice_rows,opts={})
      ret = Array.new
      attr_link_updates = Array.new
      id_list = array_slice_rows.map{|r|r[:id]}
      Model.select_process_and_update(attr_mh,[:id,:value_derived],id_list) do |existing_vals|
        ndx_existing_vals = existing_vals.inject({}){|h,r|h.merge(r[:id] => r[:value_derived])}
        attr_updates = array_slice_rows.map do |r|
          attr_id = r[:id]
          existing_val = ndx_existing_vals[attr_id]||[]
          offset = existing_val.size
          last_el = r[:array_slice].size-1
          index_map = r[:output_is_array] ?
          AttributeLink::IndexMap.generate_from_bounds(0,last_el,offset) :
            AttributeLink::IndexMap.generate_for_output_scalar(last_el,offset) 
          attr_link_update = {
            :id => r[:attr_link_id],
            :index_map => index_map
{'fraction_non_alphanumeric': 0.09724799010513296, 'fraction_numerical': 0.0006184291898577613, 'mean_word_length': 3.217079530638853, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 15, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5895375', 'n_tokens_mistral': 2196, 'n_tokens_neox': 2123, 'n_words': 377}