<commit_before>module SfdcBulkQueryConfig
    require "csv"
    extend ActiveSupport::Concern

    # methods defined here are going to extend the class, not the instance of it
    module ClassMethods
      @sobject = nil
      @swhere = nil
      @slimit = nil


      def base_class
        a = self.name.split("::")
        a.pop
        a.join("::").constantize
      end

      def set_sfdc_object(name)
        @sobject = name
      end

      def set_sfdc_where(where)
        @swhere = where
      end

      def set_sfdc_limit(limit)
        @slimit = limit
      end

      def set_sfdc_refresh_method(method)
        @srefresh_method = method
      end


      def sobject
        @sobject || self.name.split("::")[1]
      end

      def sfdc_where
        @swhere 
      end

      def sfdc_limit
        @slimit
      end

      def sfdc_refresh_method
        @srefresh_method ||= :upsert
      end

    end

    def querier_class
      self.class
    end
    
    def target_class
      self.class.base_class
    end


    def excluded_attributes=(exc)
      @excluded_attributes = [exc].flatten
    end
    
    def excluded_attributes
      @excluded_attributes || []
    end



    def initialize_target_row(target_row,sfdc_row)
      mapping = self.mapping
      ch = target_class.columns_hash
      sfdc_row.each do |k,v|
        cinfo = ch[k]
        next if cinfo.nil?
        target_name = mapping[k]
        v = nil if !v.nil? && v.empty?
        target_row[target_name] = v
      end
      return target_row
    end

    def reload_from_sfdc(method: )
      if method.nil?
        method = self.class.sfdc_refresh_method
      end
      filename = self.run
      self.import(filename,method)
    end

    def import(filename,method)
      if method.nil?
        method = self.class.sfdc_refresh_method
      end
      if method.to_sym == :truncate
        target_class.connection.execute(truncate_sql_statement(table_name: target_class.table_name))
      end
      
      csv = CSV.open(filename,headers:true,header_converters:lambda { |h| h.downcase})
      csv.each do |row|
        if method == :truncate
          target_row = target_class.new
        else
          target_row = target_class.find_by(self.mapping["id"] => row["id"]) || target_class.new
        end
        initialize_target_row(target_row,row)
        target_row.save
      end
      true
    end

    def truncate_sql_statement(params)
<<-SQL
TRUNCATE TABLE #{params[:table_name]};
SQL
    end




end<commit_msg>fixed: bypass error during import<commit_after>module SfdcBulkQueryConfig
    require "csv"
    extend ActiveSupport::Concern

    # methods defined here are going to extend the class, not the instance of it
    module ClassMethods
      @sobject = nil
      @swhere = nil
      @slimit = nil


      def base_class
        a = self.name.split("::")
        a.pop
        a.join("::").constantize
      end

      def set_sfdc_object(name)
        @sobject = name
      end

      def set_sfdc_where(where)
        @swhere = where
      end

      def set_sfdc_limit(limit)
        @slimit = limit
      end

      def set_sfdc_refresh_method(method)
        @srefresh_method = method
      end


      def sobject
        @sobject || self.name.split("::")[1]
      end

      def sfdc_where
        @swhere 
      end

      def sfdc_limit
        @slimit
      end

      def sfdc_refresh_method
        @srefresh_method ||= :upsert
      end

    end

    def querier_class
      self.class
    end
    
    def target_class
      self.class.base_class
    end


    def excluded_attributes=(exc)
      @excluded_attributes = [exc].flatten
    end
    
    def excluded_attributes
      @excluded_attributes || []
    end



    def initialize_target_row(target_row,sfdc_row)
      mapping = self.mapping
      ch = target_class.columns_hash
      sfdc_row.each do |k,v|
        cinfo = ch[k]
        next if cinfo.nil?
        target_name = mapping[k]
        v = nil if !v.nil? && v.empty?
        target_row[target_name] = v
      end
      return target_row
    end

    def reload_from_sfdc(method: )
      if method.nil?
        method = self.class.sfdc_refresh_method
      end
      filename = self.run
      self.import(filename,method)
    end

    def import(filename,method)
      if method.nil?
        method = self.class.sfdc_refresh_method
      end
      if method.to_sym == :truncate
        target_class.connection.execute(truncate_sql_statement(table_name: target_class.table_name))
      end
      
      csv = CSV.open(filename,headers:true,header_converters:lambda { |h| h.downcase})
      csv.each do |row|
        if method == :truncate
          target_row = target_class.new
        else
          target_row = target_class.find_by(self.mapping["id"] => row["id"]) || target_class.new
        end
        initialize_target_row(target_row,row)
        begin
          target_row.save
        rescue => e
          Rails.logger.info("#{e.inspect} Cant import #{row} ")
        end
      end
      true
    end

    def truncate_sql_statement(params)
<<-SQL
TRUNCATE TABLE #{params[:table_name]};
SQL
    end




end
{'fraction_non_alphanumeric': 0.08226620100892511, 'fraction_numerical': 0.00038804811796662784, 'mean_word_length': 1.9866743916570104, 'pattern_counts': {'":': 6, '<': 7, '<?xml version=': 0, '>': 6, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3008247', 'n_tokens_mistral': 1693, 'n_tokens_neox': 1606, 'n_words': 359}