<commit_before>class String

  # If the value os a column is already a String and it calls to_hstore, it
  # just returns self. Validation occurs afterwards.
  def to_hstore
    self
  end

  # Validates the hstore format. Valid formats are:
  # * An empty string
  # * A string like %("foo"=>"bar"). I'll call it a "double quoted hstore format".
  # * A string like %(foo=>bar). Postgres doesn't emit this but it does accept it as input, we should accept any input Postgres does

  def valid_hstore?
    pair = hstore_pair
    !!match(/^\s*(#{pair}\s*(,\s*#{pair})*)?\s*$/)
  end

  # Test to see if a string is actually a number
  # test shamelessly stolen from: http://railsforum.com/viewtopic.php?id=19081
  def is_numeric?
    begin Float(self)
      true 
    end
    rescue
      false
  end

  def to_implicit_type(can_be_obj=true)
    return true if self =~ /^true$/i
    return false if self =~ /^false$/i
    return nil if self =~ /^NULL$/i
    return self.to_f if self.is_numeric?
    unquoted = self[1..-2]
    return $1.gsub(/\\(.)/, '\1').from_hstore if can_be_obj && unquoted =~ /^\{\\(.*?)\}/
    unquoted
  end

  # Creates a hash from a valid double quoted hstore format, 'cause this is the format
  # that postgresql spits out.
  def from_hstore
    token_pairs = (scan(hstore_pair)).map { |k,v| [k.to_implicit_type(false), v.to_implicit_type] }
    Hash[ token_pairs ]
  end

  private

  def hstore_pair
    quoted_string = /"[^"\\]*(?:\\.[^"\\]*)*"/
    unquoted_string = /[^\s=,][^\s=,\\]*(?:\\.[^\s=,\\]*|=[^,>])*/
    string = /(#{quoted_string}|#{unquoted_string})/
    /#{string}\s*=>\s*#{string}/
  end
end<commit_msg>correctly return a float or an int if numeric unquoted value<commit_after>class String

  # If the value os a column is already a String and it calls to_hstore, it
  # just returns self. Validation occurs afterwards.
  def to_hstore
    self
  end

  # Validates the hstore format. Valid formats are:
  # * An empty string
  # * A string like %("foo"=>"bar"). I'll call it a "double quoted hstore format".
  # * A string like %(foo=>bar). Postgres doesn't emit this but it does accept it as input, we should accept any input Postgres does

  def valid_hstore?
    pair = hstore_pair
    !!match(/^\s*(#{pair}\s*(,\s*#{pair})*)?\s*$/)
  end

  # Test to see if a string is actually a number
  # test shamelessly stolen from: http://railsforum.com/viewtopic.php?id=19081
  def is_numeric?
    begin Float(self)
      true 
    end
    rescue
      false
  end

  def to_implicit_type(can_be_obj=true)
    return true if self =~ /^true$/i
    return false if self =~ /^false$/i
    return nil if self =~ /^NULL$/i
    if self.is_numeric?
      if self =~ /^{0,9}\.{0,9}$/
        return self.to_f
      else
        return self.to_i
      end
    end
    unquoted = self[1..-2]
    return $1.gsub(/\\(.)/, '\1').from_hstore if can_be_obj && unquoted =~ /^\{\\(.*?)\}/
    unquoted
  end

  # Creates a hash from a valid double quoted hstore format, 'cause this is the format
  # that postgresql spits out.
  def from_hstore
    token_pairs = (scan(hstore_pair)).map { |k,v| [k.to_implicit_type(false), v.to_implicit_type] }
    Hash[ token_pairs ]
  end

  private

  def hstore_pair
    quoted_string = /"[^"\\]*(?:\\.[^"\\]*)*"/
    unquoted_string = /[^\s=,][^\s=,\\]*(?:\\.[^\s=,\\]*|=[^,>])*/
    string = /(#{quoted_string}|#{unquoted_string})/
    /#{string}\s*=>\s*#{string}/
  end
end
{'fraction_non_alphanumeric': 0.16031422752400348, 'fraction_numerical': 0.006400931044515566, 'mean_word_length': 2.8327759197324416, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 11, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1678742', 'n_tokens_mistral': 1281, 'n_tokens_neox': 1233, 'n_words': 361}