<commit_before>require 'pp'
Encoding.default_external = Encoding::UTF_8

module GitMedia
  module Status

    def self.run!(opts)
      @server = GitMedia.get_transport
      r = self.get_pull_status(opts[:dir])
      c = self.get_push_status
      self.print_pull_status(r, opts[:long], opts[:dir])
      self.print_push_status(c, opts[:long])
    end

    def self.get_pull_status(relative_path=false, server=@server)
      # Find files that are likely media entries and check if they are
      # downloaded already
      references = {:unpulled => [], :pulled => [], :deleted => [], :not_on_server => []}
      if relative_path
        files = `git ls-tree -l -r HEAD | tr "\\000" \\\\n`.split("\n")
        repo_path = "."
      else
        files = `git ls-tree -l -r HEAD --full-tree | tr "\\000" \\\\n`.split("\n")
        repo_path = `git rev-parse --show-toplevel`.chomp
      end
      files = files.map { |f| s = f.split("\t"); [s[0].split(' ').last, s[1]] }
      # => files = [[file_size, file_name], [...], ...]
      # Find unpulled files after looking at its size
      # TODO: this seems a little bit risky, what if a file has mistakenly the
      # same size
      files = files.select { |f| f[0] == '41' } # it's the right size
      files.each do |tree_size, fname|
        fname = File.join(repo_path, fname)
        if File.exists?(fname)
          size = File.size(fname)
          # Windows newlines can offset file size by 1
          if size == tree_size.to_i or size == tree_size.to_i + 1
            # TODO: read in the data and verify that it's a sha + newline
            fname = fname.tr("\\","") #remove backslash
            sha = File.read(fname).strip
            if sha.length == 40 && sha =~ /^[0-9a-f]+$/
              references[:unpulled] << [fname, sha]
              references[:not_on_server] << fname if !server.exist?(sha)
            end
          else
            references[:pulled] << fname
          end
        else
          # File was deleted
          references[:deleted] << fname
        end
      end
      references
    end

    def self.get_push_status(server=@server)
      # Find files in media buffer and check if they are uploaded already
      references = {:unpushed => [], :pushed => []}
      all_cache = Dir.chdir(GitMedia.get_media_buffer) { Dir.glob('*') }
      unpushed_files = server.get_unpushed(all_cache) || []
      references[:unpushed] = unpushed_files
      references[:pushed] = all_cache - unpushed_files rescue []
      references
    end

    def self.print_pull_status(refs, long=false, relative_path=false)

      # Unpulled media
      if refs[:unpulled].size > 0
        hint = ", run 'git media pull"
        hint << " --dir" if relative_path
        hint << "' to download them"
      else
        hint = ""
      end
      if refs[:not_on_server].size > 0
        hint << ". WARNING: " + refs[:not_on_server].size.to_s + " of them are not on the server!"
      end
      puts "== Unpulled Media: " + refs[:unpulled].size.to_s + " file(s)" + hint
      if long
        refs[:unpulled].each do |file, sha|
          # TODO: get local file name
          puts "   " + sha[0, 8] + " " + file
        end
        puts
      end

      # Pulled media
      puts "== Pulled Media:   " + refs[:pulled].size.to_s + " file(s)"
      if long
        refs[:pulled].each do |file|
          size = File.size(file)
          puts "   " + "(#{self.to_human(size)})".ljust(8) + " #{file}"
        end
        puts
      end

      # Deleted media
      if refs[:deleted].size > 0
        puts "== Deleted Media:  " + refs[:deleted].size.to_s + " file(s)"
        if long
          refs[:deleted].each do |file|
            puts "           " + " #{file}"
          end
          puts
        end
      end

    end

    def self.print_push_status(refs, long=false)

      # Unpushed media<commit_msg>Add hint how to remove deleted files completely<commit_after>require 'pp'
Encoding.default_external = Encoding::UTF_8

module GitMedia
  module Status

    def self.run!(opts)
      @server = GitMedia.get_transport
      r = self.get_pull_status(opts[:dir])
      c = self.get_push_status
      self.print_pull_status(r, opts[:long], opts[:dir])
      self.print_push_status(c, opts[:long])
    end

    def self.get_pull_status(relative_path=false, server=@server)
      # Find files that are likely media entries and check if they are
      # downloaded already
      references = {:unpulled => [], :pulled => [], :deleted => [], :not_on_server => []}
      if relative_path
        files = `git ls-tree -l -r HEAD | tr "\\000" \\\\n`.split("\n")
        repo_path = "."
      else
        files = `git ls-tree -l -r HEAD --full-tree | tr "\\000" \\\\n`.split("\n")
        repo_path = `git rev-parse --show-toplevel`.chomp
      end
      files = files.map { |f| s = f.split("\t"); [s[0].split(' ').last, s[1]] }
      # => files = [[file_size, file_name], [...], ...]
      # Find unpulled files after looking at its size
      # TODO: this seems a little bit risky, what if a file has mistakenly the
      # same size
      files = files.select { |f| f[0] == '41' } # it's the right size
      files.each do |tree_size, fname|
        fname = File.join(repo_path, fname)
        if File.exists?(fname)
          size = File.size(fname)
          # Windows newlines can offset file size by 1
          if size == tree_size.to_i or size == tree_size.to_i + 1
            # TODO: read in the data and verify that it's a sha + newline
            fname = fname.tr("\\","") #remove backslash
            sha = File.read(fname).strip
            if sha.length == 40 && sha =~ /^[0-9a-f]+$/
              references[:unpulled] << [fname, sha]
              references[:not_on_server] << fname if !server.exist?(sha)
            end
          else
            references[:pulled] << fname
          end
        else
          # File was deleted
          references[:deleted] << fname
        end
      end
      references
    end

    def self.get_push_status(server=@server)
      # Find files in media buffer and check if they are uploaded already
      references = {:unpushed => [], :pushed => []}
      all_cache = Dir.chdir(GitMedia.get_media_buffer) { Dir.glob('*') }
      unpushed_files = server.get_unpushed(all_cache) || []
      references[:unpushed] = unpushed_files
      references[:pushed] = all_cache - unpushed_files rescue []
      references
    end

    def self.print_pull_status(refs, long=false, relative_path=false)

      # Unpulled media
      if refs[:unpulled].size > 0
        hint = ", run 'git media pull"
        hint << " --dir" if relative_path
        hint << "' to download them"
      else
        hint = ""
      end
      if refs[:not_on_server].size > 0
        hint << ". WARNING: " + refs[:not_on_server].size.to_s + " of them are not on the server!"
      end
      puts "== Unpulled Media: " + refs[:unpulled].size.to_s + " file(s)" + hint
      if long
        refs[:unpulled].each do |file, sha|
          # TODO: get local file name
          puts "   " + sha[0, 8] + " " + file
        end
        puts
      end

      # Pulled media
      puts "== Pulled Media:   " + refs[:pulled].size.to_s + " file(s)"
      if long
        refs[:pulled].each do |file|
          size = File.size(file)
          puts "   " + "(#{self.to_human(size)})".ljust(8) + " #{file}"
        end
        puts
      end

      # Deleted media
      if refs[:deleted].size > 0
        hint = ", run 'git rm <file(s)> && git commit' to remove completely"
        puts "== Deleted Media:  " + refs[:deleted].size.to_s + " file(s)" + hint
        if long
          refs[:deleted].each do |file|
            puts "           " + " #{file}"
          end
          puts
        end
      end

    end

    def self.print_push_status(refs, long=false)

      # Unpushed media
{'fraction_non_alphanumeric': 0.13456162642947903, 'fraction_numerical': 0.006099110546378653, 'mean_word_length': 1.9735549678881752, 'pattern_counts': {'":': 0, '<': 32, '<?xml version=': 0, '>': 24, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1323252', 'n_tokens_mistral': 2560, 'n_tokens_neox': 2517, 'n_words': 671}