<commit_before>require 'libxml'
require 'open-uri'
require 'ostruct'
require 'active_support/all'

unless Object.respond_to?(:blank)
  class Object
    def blank?
      respond_to?(:empty?) ? empty? : !self
    end
  end
end

unless Object.respond_to?(:to_bool)
  class Object
    def to_bool
      return false if self == false || self.blank? || self.to_s =~ (/(false|f|no|n|0)$/i)
      return true if self == true || self.to_s =~ (/(true|t|yes|y|1)$/i)
      raise ArgumentError.new("invalid value for Boolean: \"#{self}\"")
    end
  end
end

unless OpenStruct.respond_to?(:to_hash)

  class OpenStruct
    def to_hash
      return {} unless self._table_hash
      return @table.inject({}) do |h, (key, value)|
        value = value.to_hash if value.respond_to?(:to_hash)
        h.merge!({ key: value })
        h
      end
    end

    private

    def _table_hash
      @table
    end

  end
end

#<commit_msg>Remove to_hash method for OpenStruct.<commit_after>require 'libxml'
require 'open-uri'
require 'ostruct'
require 'active_support/all'

unless Object.respond_to?(:blank)
  class Object
    def blank?
      respond_to?(:empty?) ? empty? : !self
    end
  end
end

unless Object.respond_to?(:to_bool)
  class Object
    def to_bool
      return false if self == false || self.blank? || self.to_s =~ (/(false|f|no|n|0)$/i)
      return true if self == true || self.to_s =~ (/(true|t|yes|y|1)$/i)
      raise ArgumentError.new("invalid value for Boolean: \"#{self}\"")
    end
  end
end

unless OpenStruct.respond_to?(:to_hash)

  class OpenStruct
    def xx_to_hash
      return {} unless self._table_hash
      return @table.inject({}) do |h, (key, value)|
        value = value.to_hash if value.respond_to?(:to_hash)
        h.merge!({ key: value })
        h
      end
    end

    private

    def _table_hash
      @table
    end

  end
end

#
{'fraction_non_alphanumeric': 0.13164965072541646, 'fraction_numerical': 0.0021493820526598604, 'mean_word_length': 2.8, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 2}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5497647', 'n_tokens_mistral': 659, 'n_tokens_neox': 645, 'n_words': 168}