<commit_before>require 'pathname'

class Pathname
  BAD_PATHS = [Pathname.new('.'), Pathname.new('..')]
  
  def to_pastie(recursive = false, full = false)
    if self.file?
      return pastie(self, full)
    elsif self.directory?
      pathnames = []
      
      self.children.each do |pathname|
        if recursive
          pathnames << pathname unless BAD_PATHS.include?(pathname)
        elsif pathname.file?
          pathnames << pathname
        end
      end
      
      return pathnames.map do |pathname|
        pathname.to_pastie(recursive, full)
      end.compact.join("\n")
    end
  end
  
  private
  
  def pastie(pathname, full = false)
    if pathname.readable?
      path = full ? pathname.realpath : pathname.basename
      section = "## "
      content = File.read(pathname).sub(/(\r|\n)+\z/, '')
      
      return "#{section}#{path}\n#{content}"
    else
      return nil
    end
  end
end
<commit_msg>trying to avoid ArgumentErrors from Ruby >= 1.9 encodings<commit_after>require 'pathname'

class Pathname
  BAD_PATHS = [Pathname.new('.'), Pathname.new('..')]
  
  def to_pastie(recursive = false, full = false)
    if self.file?
      return pastie(self, full)
    elsif self.directory?
      pathnames = []
      
      self.children.each do |pathname|
        if recursive
          pathnames << pathname unless BAD_PATHS.include?(pathname)
        elsif pathname.file?
          pathnames << pathname
        end
      end
      
      return pathnames.map do |pathname|
        pathname.to_pastie(recursive, full)
      end.compact.join("\n")
    end
  end
  
  private
  
  def pastie(pathname, full = false)
    if pathname.readable?
      path = full ? pathname.realpath : pathname.basename
      section = "## "
      content = begin
        File.read(pathname).sub(/(\r|\n)+\z/, '')
      rescue ArgumentError
        File.read(pathname, :encoding => "BINARY").sub(/(\r|\n)+\z/, '')
      end
      
      return "#{section}#{path}\n#{content}"
    end
  end
end

{'fraction_non_alphanumeric': 0.125249500998004, 'fraction_numerical': 0.000998003992015968, 'mean_word_length': 2.1875993640699525, 'pattern_counts': {'":': 0, '<': 11, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6666224', 'n_tokens_mistral': 662, 'n_tokens_neox': 651, 'n_words': 143}