<commit_before>module Harbor
  class FileStore
    class Local < Harbor::FileStore

      attr_accessor :path, :options

      def initialize(path, options = {})
        @path = Pathname(path)

        @options = options

        ::FileUtils.mkdir_p(path.to_s)
      end

      def get(path)
        Harbor::FileStore::File.new(self, path)
      end

      def put(path, file)
        f = Harbor::FileStore::File.new(self, path)

        size = file.is_a?(::File) ? ::File.size(file) : file.size
        cleanup(size)

        ::FileUtils.mkdir_p((@path + path).parent.to_s) unless (@path + path).parent.exist?

        while data = file.read(500_000)
          f.write data
        end

        f.write nil
      end

      def delete(path)
        ::FileUtils.rm(@path + path)
        Harbor::File.rmdir_p((@path + path).parent.to_s)
      end

      def exists?(path)
        (@path + path).exist?
      end

      def open(path, mode = "r", &block)
        ::File.open(@path + path, mode, &block)
      end

      def size(path)
        ::File.size(@path + path)
      end

      def local?
        true
      end

      private

      def __size__
        `du -sk #{Shellwords.escape(@path.to_s)} | awk '{ print $1; }'`.chomp.to_i * 1024
      end

      def cleanup(space = nil)
        size = __size__
        size -= space if space # make room for a specified file size if specified

        return false unless @options[:cache_size] && size > @options[:cache_size]

        Dir[@path + "**/*"].each do |file|
          next unless ::File.ctime(file) < Time.now - (@options[:cache_time] || 60)

          file = Pathname(file).relative_path_from(@path)

          size -= size(file)
          delete(file)<commit_msg>Sort by ctime and reject directories when traversing cache.<commit_after>module Harbor
  class FileStore
    class Local < Harbor::FileStore

      attr_accessor :path, :options

      def initialize(path, options = {})
        @path = Pathname(path)

        @options = options

        ::FileUtils.mkdir_p(path.to_s)
      end

      def get(path)
        Harbor::FileStore::File.new(self, path)
      end

      def put(path, file)
        f = Harbor::FileStore::File.new(self, path)

        size = file.is_a?(::File) ? ::File.size(file) : file.size
        cleanup(size)

        ::FileUtils.mkdir_p((@path + path).parent.to_s) unless (@path + path).parent.exist?

        while data = file.read(500_000)
          f.write data
        end

        f.write nil
      end

      def delete(path)
        ::FileUtils.rm(@path + path)
        Harbor::File.rmdir_p((@path + path).parent.to_s)
      end

      def exists?(path)
        (@path + path).exist?
      end

      def open(path, mode = "r", &block)
        ::File.open(@path + path, mode, &block)
      end

      def size(path)
        ::File.size(@path + path)
      end

      def local?
        true
      end

      private

      def __size__
        `du -sk #{Shellwords.escape(@path.to_s)} | awk '{ print $1; }'`.chomp.to_i * 1024
      end

      def cleanup(space = nil)
        size = __size__
        size -= space if space # make room for a specified file size if specified

        return false unless @options[:cache_size] && size > @options[:cache_size]

        files = Dir[@path + "**/*"].select { |f| ::File.file?(f) }.sort_by { |f| ::File.ctime(f) }
        files.each do |file|
          next unless ::File.ctime(file) < Time.now - (@options[:cache_time] || 60)

          file = Pathname(file).relative_path_from(@path)

          size -= size(file)
          delete(file)
{'fraction_non_alphanumeric': 0.133761519128735, 'fraction_numerical': 0.007260541748115051, 'mean_word_length': 2.012615643397813, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5191856', 'n_tokens_mistral': 1174, 'n_tokens_neox': 1114, 'n_words': 276}