<commit_before>module Tori
  module Backend
    class FileSystem
      attr_accessor :root
      def initialize(root)
        @root = root
        FileUtils.mkdir_p @root.to_s
      end

      def write(filename, resource, opts = nil)
        case resource
        when String
          ::File.open(path(filename), 'w'){ |f| f.write resource }
        when Pathname
          # see also https://bugs.ruby-lang.org/issues/11199
          ::File.open(resource) { |src|
            FileUtils.mkdir_p path(filename).dirname
            ::File.open(path(filename), 'w'){ |dst|
              ::IO.copy_stream src, dst
            }
          }
        else
          ::File.open(path(filename), 'w') do |dst|
            ::IO.copy_stream resource, dst
          end
        end
      end

      def delete(filename)
        ::File.unlink path(filename)
      end

      def exist?(filename)
        ::File.exist? path(filename)
      end
      alias exists? exist?

      def read(filename)
        ::File.read path(filename)<commit_msg>Support bin resource<commit_after>module Tori
  module Backend
    class FileSystem
      attr_accessor :root
      def initialize(root)
        @root = root
        FileUtils.mkdir_p @root.to_s
      end

      def write(filename, resource, opts = nil)
        case resource
        when String
          ::File.open(path(filename), 'wb'){ |f| f.write resource }
        when Pathname
          # see also https://bugs.ruby-lang.org/issues/11199
          ::File.open(resource) { |src|
            FileUtils.mkdir_p path(filename).dirname
            ::File.open(path(filename), 'wb'){ |dst|
              ::IO.copy_stream src, dst
            }
          }
        else
          ::File.open(path(filename), 'wb') do |dst|
            ::IO.copy_stream resource, dst
          end
        end
      end

      def delete(filename)
        ::File.unlink path(filename)
      end

      def exist?(filename)
        ::File.exist? path(filename)
      end
      alias exists? exist?

      def read(filename)
        ::File.read(path(filename), mode: 'rb')
{'fraction_non_alphanumeric': 0.11031175059952038, 'fraction_numerical': 0.004796163069544364, 'mean_word_length': 1.7519788918205805, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3197201', 'n_tokens_mistral': 621, 'n_tokens_neox': 599, 'n_words': 146}