<commit_before>require 'active_support/core_ext'

class Comparator
  
  def self.normalize_value(val)
    return val if val.is_a?(TimeDateMatcher)
    val = ActiveSupport::JSON.decode(val) if val.is_a?(String)
    val = self.stringify_keys(val) if val.is_a?(Hash)
    val = val.collect{|v| v.is_a?(Hash) || n.is_a?(Array) ? self.normalize_value(v) : v} if val.is_a?(Array)
    val
  end
  
  def self.diff(obj1, obj2)
    return {} if obj1 == obj2
    return self.array_diff(obj1, obj2) if obj1.is_a?(Array) && obj2.is_a?(Array)
    return self.hash_diff(obj1, obj2) if obj1.is_a?(Hash) && obj2.is_a?(Hash)
    return [obj1, obj2]
  end
  
  def self.array_diff(obj1, obj2)
    {}.tap do |ret|
      obj1.each_index do |k|
        ret[k] = self.diff(obj1[k], obj2[k]) unless obj1[k] == obj2[k]
      end
    end
  end
  
  def self.hash_diff(obj1, obj2)
    obj1, obj2 = self.stringify_keys(obj1), self.stringify_keys(obj2)
    (obj1.keys + obj2.keys).uniq.inject({}) do |memo, key|
      if !obj1.keys.include?(key) 
        memo[key] = ["KEY MISSING", obj2[key]]
      elsif !obj2.keys.include?(key) 
        memo[key] = [obj1[key], "KEY MISSING"]
      elsif obj1[key] != obj2[key]
        memo[key] = self.diff(obj1[key], obj2[key])
      end
      memo
    end
  end
  
  def self.stringify_keys(hash_or_array)
    return hash_or_array.collect{|v| v.is_a?(Hash) || v.is_a?(Array) ? self.stringify_keys(v) : v} if hash_or_array.is_a?(Array)
    {}.tap do |ret|
      hash_or_array.each_pair.each do |k, v|
        ret[k.to_s] = v.is_a?(Hash) || v.is_a?(Array) ? self.stringify_keys(v) : v
      end
    end
  end<commit_msg>Added 'KEY MISSING' message to missing fields in array_diff.  Fixed error where some missing fields in array_diff were not displayed because the iteration occurred over the first array's indices, even when the second array was larger, leaving elements that were in the second array, but not the first, out of the diff.<commit_after>require 'active_support/core_ext'

class Comparator
  
  def self.normalize_value(val)
    return val if val.is_a?(TimeDateMatcher)
    val = ActiveSupport::JSON.decode(val) if val.is_a?(String)
    val = self.stringify_keys(val) if val.is_a?(Hash)
    val = val.collect{|v| v.is_a?(Hash) || n.is_a?(Array) ? self.normalize_value(v) : v} if val.is_a?(Array)
    val
  end
  
  def self.diff(obj1, obj2)
    return {} if obj1 == obj2
    return self.array_diff(obj1, obj2) if obj1.is_a?(Array) && obj2.is_a?(Array)
    return self.hash_diff(obj1, obj2) if obj1.is_a?(Hash) && obj2.is_a?(Hash)
    return [obj1, "KEY MISSING"] if obj2.nil?
    return ["KEY MISSING", obj2] if obj1.nil?
    return [obj1, obj2]
  end
  
  def self.array_diff(obj1, obj2)
    {}.tap do |ret|
      obj1.size >= obj2.size ? bigger_arr = obj1 : bigger_arr = obj2
      bigger_arr.each_index do |k|
        ret[k] = self.diff(obj1[k], obj2[k]) unless obj1[k] == obj2[k]
      end   
    end
  end
  
  def self.hash_diff(obj1, obj2)
    obj1, obj2 = self.stringify_keys(obj1), self.stringify_keys(obj2)
    (obj1.keys + obj2.keys).uniq.inject({}) do |memo, key|
      if !obj1.keys.include?(key) 
        memo[key] = ["KEY MISSING", obj2[key]]
      elsif !obj2.keys.include?(key) 
        memo[key] = [obj1[key], "KEY MISSING"]
      elsif obj1[key] != obj2[key]
        memo[key] = self.diff(obj1[key], obj2[key])
      end
      memo
    end
  end
  
  def self.stringify_keys(hash_or_array)
    return hash_or_array.collect{|v| v.is_a?(Hash) || v.is_a?(Array) ? self.stringify_keys(v) : v} if hash_or_array.is_a?(Array)
    {}.tap do |ret|
      hash_or_array.each_pair.each do |k, v|
        ret[k.to_s] = v.is_a?(Hash) || v.is_a?(Array) ? self.stringify_keys(v) : v
      end
    end
  end
{'fraction_non_alphanumeric': 0.14553738943982847, 'fraction_numerical': 0.021709997319753416, 'mean_word_length': 3.1512791991101223, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 4, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3258888', 'n_tokens_mistral': 1479, 'n_tokens_neox': 1507, 'n_words': 324}