<commit_before># encoding: UTF-8

require 'tms/space'

module Tms
  class Comparison
    attr_reader :backup_a, :backup_b, :total
    def initialize(backup_a, backup_b)
      @backup_a, @backup_b = backup_a, backup_b
    end

    def run
      @total = 0
      begin
        root_dirs = (backup_a.path.children(false) | backup_b.path.children(false)).sort
        root_dirs.reject!{ |root_dir| root_dir.path[0, 1] == '.' }
        root_dirs.each do |root_dir|
          dirs = Backup.filter_dirs? ? Backup.filter_dirs.map{ |filter_dir| root_dir / filter_dir } : [root_dir]
          dirs.each do |dir|
            compare(backup_a.path / dir, backup_b.path / dir, Path.new('/', dir))
          end
        end
      rescue Interrupt
        puts
        puts 'Interrupted'
      end
      line "#{colorize 'Total:', :total} #{space(total)}"
    end

  private

    def compare(a, b, path)
      case
      when !a.exist?
        diff_line b, path, colorize('  █', :right), "#{path}#{b.postfix}", :recursive => true
      when !b.exist?
        diff_line a, path, colorize('█  ', :left), "#{path}#{a.postfix}", :recursive => true, :no_total => true
      when a.ftype != b.ftype
        diff_line b, path, colorize('█≠█', :diff_type), "#{path}#{b.postfix} (#{a.ftype}=>#{b.ftype})", :recursive => true
      when a.lstat.ino != b.lstat.ino
        if a.readable_real? && b.readable_real?
          diff_line b, path, colorize('█≠█', :diff), "#{path}#{b.postfix}" unless b.symlink? && a.readlink == b.readlink
          if b.directory? && !b.symlink?
            (a.children(false) | b.children(false)).sort.each do |child|
              compare(a / child, b / child, path / child)
            end
          end
        else
          line colorize("??? #{Space::NOT_COUNTED_SPACE} #{path}#{a.postfix}", :unreadable)
        end
      else
        progress do
          path
        end
      end
    end

    COLORS = {
      :total => {:extra => :bold},
      :right => {:foreground => :green},
      :left => {:foreground => :blue},
      :diff_type => {:foreground => :red, :extra => :bold},
      :diff => {:foreground => :yellow},
      :unreadable => {:foreground => :red, :extra => :bold},
    }
    CLEAR_LINE = "\e[K"

    def line(s)
      $stdout.puts "#{s}#{CLEAR_LINE}"
    end

    def trim_right_colored(s, length)
      length = 0 if length < 0
      colorizers = []
      s.gsub(/\e\[\d+(;\d+)*m/) do
        if $`.length < length
          length += $&.length
        else
          colorizers << $&
        end
      end
      s[0, length] << colorizers.join('')
    end

    def trim_left_colored(s, length)
      length = 0 if length < 0
      colorizers = []
      s.reverse.gsub(/m(\d+;)*\d+\[\e/) do
        if $`.length < length
          length += $&.length
        else
          colorizers << $&.reverse
        end
      end
      length = s.length if length > s.length
      colorizers.reverse.join('') << s[-length, length]
    end

    def progress
      if Tms::Backup.show_progress?
        @last_progress ||= Time.now
        if (now = Time.now) > @last_progress + 0.1
          line = yield.to_s
          if width = terminal_width
            line = trim_left_colored(line, terminal_width - 1)
          end
          $stderr.print "#{line}#{CLEAR_LINE}\r"
          @last_progress = now
        end
      end
    end

    def space(size)
      Tms::Space.space(size, :color => Tms::Backup.colorize?)
    end

    def colorize(s, type)
      if Tms::Backup.colorize?
        Colored.colorize(s, COLORS[type])
      else
        s
      end
    end

    def diff_line(backup_path, path, prefix, postfix, options = {})
      sub_total = count_space(backup_path, options[:recursive])
      line "#{prefix} #{space sub_total} #{postfix}"
      unless options[:no_total]
        @total += sub_total
      end
    end

    def count_space(backup_path, recursive)
      sub_total = 0
      if recursive
        backup_path.find do |sub_path|
          sub_total += sub_path.size_if_real_file
          progress do
            "#{prefix} #{space sub_total} #{path / sub_path.to_s[backup_path.to_s.length..-1].to_s}"
          end
        end<commit_msg>diff_line gets both paths, count both totals, count_space also should get path and prefix<commit_after># encoding: UTF-8

require 'tms/space'

module Tms
  class Comparison
    attr_reader :backup_a, :backup_b
    def initialize(backup_a, backup_b)
      @backup_a, @backup_b = backup_a, backup_b
    end

    def run
      @a_total, @b_total = 0, 0
      begin
        root_dirs = (backup_a.path.children(false) | backup_b.path.children(false)).sort
        root_dirs.reject!{ |root_dir| root_dir.path[0, 1] == '.' }
        root_dirs.each do |root_dir|
          dirs = Backup.filter_dirs? ? Backup.filter_dirs.map{ |filter_dir| root_dir / filter_dir } : [root_dir]
          dirs.each do |dir|
            compare(backup_a.path / dir, backup_b.path / dir, Path.new('/', dir))
          end
        end
      rescue Interrupt
        puts
        puts 'Interrupted'
      end
      line [colorize('Total:', :total), space(@b_total)].join(' ')
    end

  private

    def compare(a, b, path)
      case
      when !a.exist?
        diff_line nil, b, path, colorize('  █', :right), "#{path}#{b.postfix}", true
      when !b.exist?
        diff_line a, nil, path, colorize('█  ', :left), "#{path}#{a.postfix}", true
      when a.ftype != b.ftype
        diff_line a, b, path, colorize('█≠█', :diff_type), "#{path}#{b.postfix} (#{a.ftype}=>#{b.ftype})", true
      when a.lstat.ino != b.lstat.ino
        if a.readable_real? && b.readable_real?
          diff_line a, b, path, colorize('█≠█', :diff), "#{path}#{b.postfix}", false unless b.symlink? && a.readlink == b.readlink
          if b.directory? && !b.symlink?
            (a.children(false) | b.children(false)).sort.each do |child|
              compare(a / child, b / child, path / child)
            end
          end
        else
          parts = ['???', Space::NOT_COUNTED_SPACE, "#{path}#{a.postfix}"]
          line colorize(parts.join(' '), :unreadable)
        end
      else
        progress do
          path
        end
      end
    end

    COLORS = {
      :total => {:extra => :bold},
      :right => {:foreground => :green},
      :left => {:foreground => :blue},
      :diff_type => {:foreground => :red, :extra => :bold},
      :diff => {:foreground => :yellow},
      :unreadable => {:foreground => :red, :extra => :bold},
    }
    CLEAR_LINE = "\e[K"

    def line(s)
      $stdout.puts "#{s}#{CLEAR_LINE}"
    end

    def trim_right_colored(s, length)
      length = 0 if length < 0
      colorizers = []
      s.gsub(/\e\[\d+(;\d+)*m/) do
        if $`.length < length
          length += $&.length
        else
          colorizers << $&
        end
      end
      s[0, length] << colorizers.join('')
    end

    def trim_left_colored(s, length)
      length = 0 if length < 0
      colorizers = []
      s.reverse.gsub(/m(\d+;)*\d+\[\e/) do
        if $`.length < length
          length += $&.length
        else
          colorizers << $&.reverse
        end
      end
      length = s.length if length > s.length
      colorizers.reverse.join('') << s[-length, length]
    end

    def progress
      if Tms::Backup.show_progress?
        @last_progress ||= Time.now
        if (now = Time.now) > @last_progress + 0.1
          l = yield.to_s
          if width = terminal_width
            l = trim_left_colored(l, terminal_width - 1)
          end
          $stderr.print "#{l}#{CLEAR_LINE}\r"
          @last_progress = now
        end
      end
    end

    def space(size)
      Tms::Space.space(size, :color => Tms::Backup.colorize?)
    end

    def colorize(s, type)
      if Tms::Backup.colorize?
        Colored.colorize(s, COLORS[type])
      else
        s
      end
    end

    def diff_line(a_path, b_path, path, prefix, postfix, recursive)
      a_sub_total = a_path ? count_space(a_path, path, prefix, recursive) : nil
      b_sub_total = b_path ? count_space(b_path, path, prefix, recursive) : nil
      line [prefix, b_sub_total ? space(b_sub_total) : space(a_sub_total), postfix].join(' ')
      @a_total += a_sub_total if a_sub_total
      @b_total += b_sub_total if b_sub_total
    end

    def count_space(backup_path, path, prefix, recursive)
      sub_total = 0
      if recursive
        backup_path.find do |sub_path|
          sub_total += sub_path.size_if_real_file
          progress do
            "#{prefix} #{space sub_total} #{path / sub_path.to_s[backup_path.to_s.length..-1].to_s}"
          end
        end
{'fraction_non_alphanumeric': 0.14537854156990246, 'fraction_numerical': 0.003367394333488156, 'mean_word_length': 2.0618556701030926, 'pattern_counts': {'":': 0, '<': 27, '<?xml version=': 0, '>': 43, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '76379', 'n_tokens_mistral': 2973, 'n_tokens_neox': 2922, 'n_words': 675}