<commit_before>module MetaContent
  class Sanitizer

    def initialize(record)
      @record = record
    end

    def from_database(raw_results)
      sanitized_results = HashWithIndifferentAccess.new
      raw_results.each do |namespace, results|
        results.each do |k,v|
          options = schema[namespace] || {}
          options = options[k]
          next unless options
          sanitized_results[namespace] ||= {}
          sanitized_results[namespace][k] = sanitize_from_database(v, options[:type] || :string)
        end
      end
      sanitized_results
    end
    
    def to_database(changes)
      sanitized_results = HashWithIndifferentAccess.new
      changes.each do |namespace, namespaced_changes|
        namespaced_changes.map do |k,v|
          options = schema[namespace] || {}
          options = options[k]
          next unless options
          
          sanitized_results[namespace] ||= {}
          sanitized_results[namespace][k] = sanitize_to_database(v, options[:type] || :string)
        end
      end
      sanitized_results
    end
    
    def sanitize(value, type)
      return nil if value.nil?
      # simulate writing and reading from database
      change = sanitize_to_database(value, type)
      sanitize_from_database(change.value, type)
    end

    protected
    
    def sanitize_from_database(value, type)
      return nil if value.nil?
      
      case type
      when :integer, :fixnum
        value.to_i
      when :float, :number
        value.to_f
      when :date
        Date.parse(value)
      when :datetime, :time
        Time.parse(value)
      when :boolean, :bool
        value == "true"
      when :symbol, :sym
        value.to_sym
      else
        value.to_s
      end
    end

    def sanitize_to_database(value, type)
      int_value = nil
      float_value = nil
      case type
      when :integer, :fixnum
        begin
          if value.respond_to? :strftime
            int_value = value.strftime("%s").to_i
          else
            int_value = value.to_i
          end
        rescue StandardError => e
          int_value = 0
        end
        str_value = int_value.to_s
      when :float, :number
        begin
          float_value = value.to_f
        rescue StandardError => e
          float_value = 0.0
        end
        str_value = float_value.to_s
      when :datetime, :time
        value = value.to_i if value.respond_to?(:to_i) && value.to_i > 0
        value = Time.at(value) if value.is_a? Integer
        
        if value.respond_to? :strftime
          str_value = value.strftime("%Y-%m-%dT%H:%M:%S%:z")
          int_value = value.strftime("%s").to_i # epoch time
        else
          str_value = nil
        end
      when :date
        value = Time.at(value).to_date if value.is_a? Integer
        
        if value.respond_to? :strftime
          str_value = value.strftime("%Y-%m-%d")
          int_value = value.strftime("%s").to_i # epoch time
        else
          str_value = nil
        end
      when :boolean, :bool
        if value != true && value != false
          value = ["1", "true", "yes"].include?(value.to_s)
        end
        
        if value == true
          str_value = "true"
          int_value = 1
        else
          str_value = "false"
          int_value = 0
        end
      when :symbol, :sym
        str_value = value.to_s
      else
        str_value = value.to_s
      end
      
      int_value   ||= float_value.to_i if float_value
      int_value   ||= str_value.length if str_value
      float_value ||= int_value.to_f
      
      Change.new(str_value, int_value, float_value)
    end

    def klass
      @record.class
    end

    def schema
      klass.meta_content_fields
    end


    class Change < Struct.new(:value, :int_value, :float_value)
      
    end
  end
end
<commit_msg>time & date can accept strings with formatted times & dates (respectively)<commit_after>module MetaContent
  class Sanitizer

    def initialize(record)
      @record = record
    end

    def from_database(raw_results)
      sanitized_results = HashWithIndifferentAccess.new
      raw_results.each do |namespace, results|
        results.each do |k,v|
          options = schema[namespace] || {}
          options = options[k]
          next unless options
          sanitized_results[namespace] ||= {}
          sanitized_results[namespace][k] = sanitize_from_database(v, options[:type] || :string)
        end
      end
      sanitized_results
    end
    
    def to_database(changes)
      sanitized_results = HashWithIndifferentAccess.new
      changes.each do |namespace, namespaced_changes|
        namespaced_changes.map do |k,v|
          options = schema[namespace] || {}
          options = options[k]
          next unless options
          
          sanitized_results[namespace] ||= {}
          sanitized_results[namespace][k] = sanitize_to_database(v, options[:type] || :string)
        end
      end
      sanitized_results
    end
    
    def sanitize(value, type)
      return nil if value.nil?
      # simulate writing and reading from database
      change = sanitize_to_database(value, type)
      sanitize_from_database(change.value, type)
    end

    protected
    
    def sanitize_from_database(value, type)
      return nil if value.nil?
      
      case type
      when :integer, :fixnum
        value.to_i
      when :float, :number
        value.to_f
      when :date
        Date.parse(value)
      when :datetime, :time
        Time.parse(value)
      when :boolean, :bool
        value == "true"
      when :symbol, :sym
        value.to_sym
      else
        value.to_s
      end
    end

    def sanitize_to_database(value, type)
      int_value = nil
      float_value = nil
      case type
      when :integer, :fixnum
        begin
          if value.respond_to? :strftime
            int_value = value.strftime("%s").to_i
          else
            int_value = value.to_i
          end
        rescue StandardError => e
          int_value = 0
        end
        str_value = int_value.to_s
      when :float, :number
        begin
          float_value = value.to_f
        rescue StandardError => e
          float_value = 0.0
        end
        str_value = float_value.to_s
      when :datetime, :time
        if value.is_a? String
          if value.to_i > 0 # string with epoch time
            value = value.to_i
          elsif Time.respond_to? :parse
            # string with formatted datetime
            value = Time.parse(value) rescue nil  # don't care if it blows up, nbd
          end
        end
        value = Time.at(value) if value.is_a? Integer
        
        if value.respond_to? :strftime
          str_value = value.strftime("%Y-%m-%dT%H:%M:%S%:z")
          int_value = value.strftime("%s").to_i # epoch time
        else
          str_value = nil
        end
      when :date
        if value.is_a? String
          if value.to_i > 0 # string with epoch time
            value = value.to_i
          elsif Date.respond_to? :parse
            # string with formatted datetime
            value = Date.parse(value) rescue nil  # don't care if it blows up, nbd
          end
        end
        value = Time.at(value).to_date if value.is_a? Integer
        
        if value.respond_to? :strftime
          str_value = value.strftime("%Y-%m-%d")
          int_value = value.strftime("%s").to_i # epoch time
        else
          str_value = nil
        end
      when :boolean, :bool
        if value != true && value != false
          value = ["1", "true", "yes"].include?(value.to_s)
        end
        
        if value == true
          str_value = "true"
          int_value = 1
        else
          str_value = "false"
          int_value = 0
        end
      when :symbol, :sym
        str_value = value.to_s
      else
        str_value = value.to_s
      end
      
      int_value   ||= float_value.to_i if float_value
      int_value   ||= str_value.length if str_value
      float_value ||= int_value.to_f
      
      Change.new(str_value, int_value, float_value)
    end

    def klass
      @record.class
    end

    def schema
      klass.meta_content_fields
    end


    class Change < Struct.new(:value, :int_value, :float_value)
      
    end
  end
end

{'fraction_non_alphanumeric': 0.08373493975903615, 'fraction_numerical': 0.0018072289156626507, 'mean_word_length': 1.767, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 10, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2735804', 'n_tokens_mistral': 2520, 'n_tokens_neox': 2473, 'n_words': 619}