<commit_before># Copyright:: (c) Autotelik Media Ltd 2012
# Author ::   <NAME>
# Date ::     March 2012
# License::   MIT
#
# Details::   The default Populator class for assigning data to models
#             
#             Provides individual population methods on an AR model.
#
#             Enables users to assign values to AR object, without knowing much about that receiving object.
#
require 'to_b'
require 'logging'

module DataShift

  Struct.new("Substitution", :pattern, :replacement)

  class Populator
    
    include DataShift::Logging

    def self.insistent_method_list
      @insistent_method_list ||= [:to_s, :to_i, :to_f, :to_b]
    end
 
    # When looking up an association, when no field provided, try each of these in turn till a match
    # i.e find_by_name, find_by_title, find_by_id
    def self.insistent_find_by_list
      @insistent_find_by_list ||= [:name, :title, :id]
    end


    # Default data embedded in column headings - so effectively apply globally
    # to teh whole column - hence class methods
    def self.set_header_default_data(operator, data )
      header_default_data[operator] = data
    end

    def self.header_default_data
      @header_default_data ||= {}
    end


    attr_reader :current_value, :original_value_before_override
    attr_reader :current_col_type
    
    attr_reader :current_attribute_hash
    attr_reader :current_method_detail
    
    def initialize   
      @current_value = nil
      @current_method_detail = nil
      @original_value_before_override = nil
      @current_attribute_hash = {}

    end
    
    # Convert DSL string forms into a hash
    # e.g
    # 
    #  "{:name => 'autechre'}" =>   Hash['name'] = autechre'
    #  "{:cost_price => '13.45', :price => 23,  :sale_price => 4.23 }"
    
    def self.string_to_hash( str )
      h = {}
      str.gsub(/[{}:]/,'').split(', ').map do |e| 
        k,v = e.split('=>')
        
        k.strip!
        v.strip!
        
        if( v.match(/['"]/) )
          h[k] = v.gsub(/["']/, '')
        elsif( v.match(/^\d+$|^\d*\.\d+$|^\.\d+$/) )
          h[k] = v.to_f
        else
          h[k] = v
        end
        h
      end
   
      h
    end
    
    # Set member variables to hold details, value and optional attributes,
    # to be set on the 'value' once created
    # 
    # Check supplied value, validate it, and if required :
    #   set to provided default value
    #   prepend any provided prefixes 
    #   add any provided postfixes
    def prepare_data(method_detail, value)

      raise NilDataSuppliedError.new("No method detail supplied for prepare_data") unless(method_detail)
     
      begin
        @prepare_data_const_regexp ||= Regexp.new( Delimiters::attribute_list_start + ".*" + Delimiters::attribute_list_end)

        if( value.is_a? ActiveRecord::Relation ) # Rails 4 - query no longer returns an array
          @current_value = value.to_a
        elsif( value.class.ancestors.include?(ActiveRecord::Base) || value.is_a?(Array))
          @current_value = value
        else
          @current_value = value.to_s
          
          attribute_hash = @current_value.slice!(@prepare_data_const_regexp)
          
          if(attribute_hash)  
            #@current_value.chop!    # the slice seems to add an extra space/eol
            @current_attribute_hash = Populator::string_to_hash( attribute_hash )
            logger.info "Populator for #{@current_value} has attributes #{@current_attribute_hash.inspect}"
          end
        end
      
        @current_attribute_hash ||= {}
       
        @current_method_detail = method_detail

        @current_col_type = @current_method_detail.col_type
      
        operator = method_detail.operator

        if(has_override?(operator))
          override_value(operator)      # takes precedence over anything else
        else
          # if no value check for a defaults from config, headers
          if(default_value(operator))
            @current_value = default_value(operator)
          elsif(Populator::header_default_data[operator])
            @current_value = Populator::header_default_data[operator].to_s
          elsif(Populator::header_default_data[operator])
            @current_value = Populator::header_default_data[operator].to_s
          elsif(method_detail.find_by_value)
            @current_value = method_detail.find_by_value
          end if(value.nil? || value.to_s.empty?)
        end

        substitute( operator )

        @current_value = "#{prefix(operator)}#{@current_value}" if(prefix(operator))
        @current_value = "#{@current_value}#{postfix(operator)}" if(postfix(operator))

      rescue => e
        logger.error("populator failed to prepare data supplied for operator #{method_detail.operator}")
        logger.error("populator stacktrace: #{e.backtrace.join('\\n')}")
        raise DataProcessingError.new("opulator failed to prepare data #{value} for operator #{method_detail.operator}")
      end
      
      return @current_value, @current_attribute_hash
    end

    # Main client hook

    def prepare_and_assign(method_detail, record, value)

      prepare_data(method_detail, value) 
       
      assign(record)
    end
    
    def assign(record)

      puts "prepare_and_assign", method_detail, record, value

      raise NilDataSuppliedError.new("No method detail - cannot assign data") unless(current_method_detail)
       
      operator = current_method_detail.operator

      logger.debug("Populator assign - [#{current_value}] via #{current_method_detail.operator} (#{current_method_detail.operator_type})")
              
      if( current_method_detail.operator_for(:belongs_to) )
 
        insistent_belongs_to(current_method_detail, record, current_value)

      elsif( current_method_detail.operator_for(:has_many) )

        # The include? check is best I can come up with right now .. to handle module/namespaces
        # TODO - can we determine the real class type of an association
        # e.g given a association taxons, which operator.classify gives us Taxon, but actually it's Spree::Taxon
        # so how do we get from 'taxons' to Spree::Taxons ? .. check if further info in reflect_on_all_associations

        begin #if(current_value.is_a?(Array) || current_value.class.name.include?(operator.classify))
          record.send(operator) << current_value<commit_msg>fixes for multiple subsitutions per field<commit_after># Copyright:: (c) Autotelik Media Ltd 2012
# Author ::   <NAME>
# Date ::     March 2012
# License::   MIT
#
# Details::   The default Populator class for assigning data to models
#             
#             Provides individual population methods on an AR model.
#
#             Enables users to assign values to AR object, without knowing much about that receiving object.
#
require 'to_b'
require 'logging'

module DataShift

  Struct.new("Substitution", :pattern, :replacement)

  class Populator
    
    include DataShift::Logging

    def self.insistent_method_list
      @insistent_method_list ||= [:to_s, :to_i, :to_f, :to_b]
    end
 
    # When looking up an association, when no field provided, try each of these in turn till a match
    # i.e find_by_name, find_by_title, find_by_id
    def self.insistent_find_by_list
      @insistent_find_by_list ||= [:name, :title, :id]
    end


    # Default data embedded in column headings - so effectively apply globally
    # to teh whole column - hence class methods
    def self.set_header_default_data(operator, data )
      header_default_data[operator] = data
    end

    def self.header_default_data
      @header_default_data ||= {}
    end


    attr_reader :current_value, :original_value_before_override
    attr_reader :current_col_type
    
    attr_reader :current_attribute_hash
    attr_reader :current_method_detail
    
    def initialize   
      @current_value = nil
      @current_method_detail = nil
      @original_value_before_override = nil
      @current_attribute_hash = {}

    end
    
    # Convert DSL string forms into a hash
    # e.g
    # 
    #  "{:name => 'autechre'}" =>   Hash['name'] = autechre'
    #  "{:cost_price => '13.45', :price => 23,  :sale_price => 4.23 }"
    
    def self.string_to_hash( str )
      h = {}
      str.gsub(/[{}:]/,'').split(', ').map do |e| 
        k,v = e.split('=>')
        
        k.strip!
        v.strip!
        
        if( v.match(/['"]/) )
          h[k] = v.gsub(/["']/, '')
        elsif( v.match(/^\d+$|^\d*\.\d+$|^\.\d+$/) )
          h[k] = v.to_f
        else
          h[k] = v
        end
        h
      end
   
      h
    end
    
    # Set member variables to hold details, value and optional attributes,
    # to be set on the 'value' once created
    # 
    # Check supplied value, validate it, and if required :
    #   set to provided default value
    #   prepend any provided prefixes 
    #   add any provided postfixes
    def prepare_data(method_detail, value)

      raise NilDataSuppliedError.new("No method detail supplied for prepare_data") unless(method_detail)
     
      begin
        @prepare_data_const_regexp ||= Regexp.new( Delimiters::attribute_list_start + ".*" + Delimiters::attribute_list_end)

        if( value.is_a? ActiveRecord::Relation ) # Rails 4 - query no longer returns an array
          @current_value = value.to_a
        elsif( value.class.ancestors.include?(ActiveRecord::Base) || value.is_a?(Array))
          @current_value = value
        else
          @current_value = value.to_s
          
          attribute_hash = @current_value.slice!(@prepare_data_const_regexp)
          
          if(attribute_hash)  
            #@current_value.chop!    # the slice seems to add an extra space/eol
            @current_attribute_hash = Populator::string_to_hash( attribute_hash )
            logger.info "Populator for #{@current_value} has attributes #{@current_attribute_hash.inspect}"
          end
        end
      
        @current_attribute_hash ||= {}
       
        @current_method_detail = method_detail

        @current_col_type = @current_method_detail.col_type
      
        operator = method_detail.operator

        if(has_override?(operator))
          override_value(operator)      # takes precedence over anything else
        else
          # if no value check for a defaults from config, headers
          if(default_value(operator))
            @current_value = default_value(operator)
          elsif(Populator::header_default_data[operator])
            @current_value = Populator::header_default_data[operator].to_s
          elsif(Populator::header_default_data[operator])
            @current_value = Populator::header_default_data[operator].to_s
          elsif(method_detail.find_by_value)
            @current_value = method_detail.find_by_value
          end if(value.nil? || value.to_s.empty?)
        end

        substitute( operator )

        @current_value = "#{prefix(operator)}#{@current_value}" if(prefix(operator))
        @current_value = "#{@current_value}#{postfix(operator)}" if(postfix(operator))

      rescue => e
        logger.error("populator failed to prepare data supplied for operator #{method_detail.operator}")
        logger.error("populator stacktrace: #{e.backtrace.join('\\n')}")
        raise DataProcessingError.new("opulator failed to prepare data #{value} for operator #{method_detail.operator}")
      end
      
      return @current_value, @current_attribute_hash
    end

    # Main client hook

    def prepare_and_assign(method_detail, record, value)

      prepare_data(method_detail, value) 
       
      assign(record)
    end
    
    def assign(record)

      raise NilDataSuppliedError.new("No method detail - cannot assign data") unless(current_method_detail)
       
      operator = current_method_detail.operator

      logger.debug("Populator assign - [#{current_value}] via #{current_method_detail.operator} (#{current_method_detail.operator_type})")
              
      if( current_method_detail.operator_for(:belongs_to) )
 
        insistent_belongs_to(current_method_detail, record, current_value)

      elsif( current_method_detail.operator_for(:has_many) )

        # The include? check is best I can come up with right now .. to handle module/namespaces
        # TODO - can we determine the real class type of an association
        # e.g given a association taxons, which operator.classify gives us Taxon, but actually it's Spree::Taxon
        # so how do we get from 'taxons' to Spree::Taxons ? .. check if further info in reflect_on_all_associations

        begin #if(current_value.is_a?(Array) || current_value.class.name.include?(operator.classify))
          record.send(operator) << current_value
{'fraction_non_alphanumeric': 0.09741331865712713, 'fraction_numerical': 0.00283041119584873, 'mean_word_length': 2.659378596087457, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 19, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '893299', 'n_tokens_mistral': 3803, 'n_tokens_neox': 3710, 'n_words': 937}