<commit_before>module StarEtl
  class DimensionFactory < Base
    
    attr_accessor :source, :sources, :dimensions
    
    def initialize
      @primary_key = StarEtl.options[:primary_key]
      @dimensions  = {}
    end
    
    def []=(k,v)
      @dimensions[k] = v
    end
    
    def run!
      [@source,@sources].flatten.compact.each do |source|
        
        get_id_range("#{source.gsub("\"",'')}_dimension")
        
        @dimensions.each_pair do |name, config|
          puts "Synchronizing #{name} from #{source}"
          
          opts = config.clone
          
          conditions = ["dimension IS NULL"]
          conditions << @id_range.call
          conditions << opts.delete(:conditions)
          join = opts.delete(:join)
          group = opts.delete(:group)
          
          cols, vals = *opts.stringify_keys.to_a.transpose
        
          insert_sql = %Q{
            INSERT INTO #{name} (#{cols.join(',')})
            SELECT #{vals.join(',')}
            FROM #{source} source
            LEFT OUTER JOIN #{name} dimension ON (#{join})
            WHERE (#{conditions.compact.join(") AND (")})
            #{"GROUP BY #{group}" if group}
          }
          
          debug insert_sql
          
          sql(insert_sql)
        
        end
      end

    end
    
    def get_id_range(source)
      get_last_id(source)
      @nothing_new = true if @last_id.to_i == @_to_id_.to_i
      @id_range = lambda {"source.#{@primary_key} BETWEEN #{@last_id} AND #{@_to_id_}"}
    end
    
    private
    
    def get_last_id(source)
      info = sql(%Q{SELECT * from etl_info WHERE table_name = '#{source}' })
      @last_id = if info.empty?
        sql(%Q{INSERT INTO etl_info (last_id, table_name) VALUES (0, '#{source}') })
        0
      else
        info.first["last_id"]
      end
      @_to_id_ = sql(%Q{SELECT max(#{@primary_key}) as "max" FROM #{source}}).first["max"]
      
      if @last_id && @_to_id_
        sql(%Q{UPDATE etl_info SET last_id = #{@_to_id_} WHERE table_name = '#{source}'})
      end
    end
    <commit_msg>transformed the source table in the wrong place<commit_after>module StarEtl
  class DimensionFactory < Base
    
    attr_accessor :source, :sources, :dimensions
    
    def initialize
      @primary_key = StarEtl.options[:primary_key]
      @dimensions  = {}
    end
    
    def []=(k,v)
      @dimensions[k] = v
    end
    
    def run!
      [@source,@sources].flatten.compact.each do |source|
        
        get_id_range(source)
        
        @dimensions.each_pair do |name, config|
          puts "Synchronizing #{name} from #{source}"
          
          opts = config.clone
          
          conditions = ["dimension IS NULL"]
          conditions << @id_range.call
          conditions << opts.delete(:conditions)
          join = opts.delete(:join)
          group = opts.delete(:group)
          
          cols, vals = *opts.stringify_keys.to_a.transpose
        
          insert_sql = %Q{
            INSERT INTO #{name} (#{cols.join(',')})
            SELECT #{vals.join(',')}
            FROM #{source} source
            LEFT OUTER JOIN #{name} dimension ON (#{join})
            WHERE (#{conditions.compact.join(") AND (")})
            #{"GROUP BY #{group}" if group}
          }
          
          debug insert_sql
          
          sql(insert_sql)
        
        end
      end

    end
    
    def get_id_range(source)
      get_last_id(source)
      @nothing_new = true if @last_id.to_i == @_to_id_.to_i
      @id_range = lambda {"source.#{@primary_key} BETWEEN #{@last_id} AND #{@_to_id_}"}
    end
    
    private
    
    def get_last_id(source)
      info = sql(%Q{SELECT * from etl_info WHERE table_name = '#{"#{source.gsub("\"",'')}_dimension"}' })
      @last_id = if info.empty?
        sql(%Q{INSERT INTO etl_info (last_id, table_name) VALUES (0, '#{"#{source.gsub("\"",'')}_dimension"}') })
        0
      else
        info.first["last_id"]
      end
      @_to_id_ = sql(%Q{SELECT max(#{@primary_key}) as "max" FROM #{source}}).first["max"]
      
      if @last_id && @_to_id_
        sql(%Q{UPDATE etl_info SET last_id = #{@_to_id_} WHERE table_name = '#{"#{source.gsub("\"",'')}_dimension"}'})
      end
    end
    
{'fraction_non_alphanumeric': 0.1377527033380348, 'fraction_numerical': 0.0009402914903620122, 'mean_word_length': 1.9103967168262654, 'pattern_counts': {'":': 0, '<': 13, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2249429', 'n_tokens_mistral': 1395, 'n_tokens_neox': 1362, 'n_words': 297}