<commit_before>if ::RUBY_VERSION >= '1.9'
  require 'ensure/encoding'
else
  require 'iconv'
end

class RemoteTable  
  class Format
    autoload :Excel, 'remote_table/format/excel'
    autoload :Excelx, 'remote_table/format/excelx'
    autoload :Delimited, 'remote_table/format/delimited'
    autoload :OpenOffice, 'remote_table/format/open_office'
    autoload :FixedWidth, 'remote_table/format/fixed_width'
    autoload :HTML, 'remote_table/format/html'
    autoload :XML, 'remote_table/format/xml'
    
    autoload :Textual, 'remote_table/format/mixins/textual'
    autoload :ProcessedByRoo, 'remote_table/format/mixins/processed_by_roo'
    autoload :ProcessedByNokogiri, 'remote_table/format/mixins/processed_by_nokogiri'
    
    attr_reader :t

    def initialize(t)
      @t = t
    end
    
    def transliterate_to_utf8(str)
      return if str.nil?
      $stderr.puts "[remote_table translit] Before:  #{str}" if ::ENV['REMOTE_TABLE_DEBUG'] and ::ENV['REMOTE_TABLE_DEBUG'].include?('translit')
      transliterated_str = if ::RUBY_VERSION >= '1.9'
        str.ensure_encoding t.properties.external_encoding, :external_encoding => t.properties.internal_encoding, :invalid_characters => :transcode
      else
        ::Iconv.conv(t.properties.external_encoding_iconv, t.properties.internal_encoding, str.to_s + ' ')[0..-2]
      end
      $stderr.puts "[remote_table translit] After:   #{transliterated_str}" if ::ENV['REMOTE_TABLE_DEBUG'] and ::ENV['REMOTE_TABLE_DEBUG'].include?('translit')
      transliterated_str
    end

    def assume_utf8(str)
      return if str.nil?
      if ::RUBY_VERSION >= '1.9'
        str.encode! t.properties.external_encoding
      else
        str
      end
    end
    
    include ::Enumerable
    def each
      raise "must be defined by format"
    end
  end
end<commit_msg>don't try to encode non-strings<commit_after>if ::RUBY_VERSION >= '1.9'
  require 'ensure/encoding'
else
  require 'iconv'
end

class RemoteTable  
  class Format
    autoload :Excel, 'remote_table/format/excel'
    autoload :Excelx, 'remote_table/format/excelx'
    autoload :Delimited, 'remote_table/format/delimited'
    autoload :OpenOffice, 'remote_table/format/open_office'
    autoload :FixedWidth, 'remote_table/format/fixed_width'
    autoload :HTML, 'remote_table/format/html'
    autoload :XML, 'remote_table/format/xml'
    
    autoload :Textual, 'remote_table/format/mixins/textual'
    autoload :ProcessedByRoo, 'remote_table/format/mixins/processed_by_roo'
    autoload :ProcessedByNokogiri, 'remote_table/format/mixins/processed_by_nokogiri'
    
    attr_reader :t

    def initialize(t)
      @t = t
    end
    
    def transliterate_to_utf8(str)
      return if str.nil?
      $stderr.puts "[remote_table translit] Before:  #{str}" if ::ENV['REMOTE_TABLE_DEBUG'] and ::ENV['REMOTE_TABLE_DEBUG'].include?('translit')
      transliterated_str = if ::RUBY_VERSION >= '1.9'
        str.ensure_encoding t.properties.external_encoding, :external_encoding => t.properties.internal_encoding, :invalid_characters => :transcode
      else
        ::Iconv.conv(t.properties.external_encoding_iconv, t.properties.internal_encoding, str.to_s + ' ')[0..-2]
      end
      $stderr.puts "[remote_table translit] After:   #{transliterated_str}" if ::ENV['REMOTE_TABLE_DEBUG'] and ::ENV['REMOTE_TABLE_DEBUG'].include?('translit')
      transliterated_str
    end

    def assume_utf8(str)
      if str.is_a?(::String) and ::RUBY_VERSION >= '1.9'
        str.encode! t.properties.external_encoding
      else
        str
      end
    end
    
    include ::Enumerable
    def each
      raise "must be defined by format"
    end
  end
end
{'fraction_non_alphanumeric': 0.11289882737932916, 'fraction_numerical': 0.00545404963185165, 'mean_word_length': 3.567870485678705, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 13, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 2}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5971664', 'n_tokens_mistral': 1283, 'n_tokens_neox': 1244, 'n_words': 238}