<commit_before># encoding: UTF-8

require 'fileutils'
require 'shellwords'

require 'dump_rake/continious_timeout'
require 'dump_rake/env'

require 'active_support/core_ext/object/blank'

Capistrano::Configuration.instance(:i_need_this!).load do
  namespace :dump do
    def dump_command(command, env = {})
      rake = env.delete(:rake) || 'rake'

      # stringify_keys! from activesupport
      DumpRake::Env.stringify!(env)

      env.update(DumpRake::Env.for_command(command, true))

      cmd = %W[-s dump:#{command}]
      cmd += env.sort.map{ |key, value| "#{key}=#{value}" }
      "#{rake} #{cmd.shelljoin}"
    end

    def fetch_rails_env
      fetch(:rails_env, 'production')
    end

    def got_rsync?
      `which rsync`
      $?.success?
    end

    def do_transfer_via(via, direction, from, to)
      case via
      when :rsync
        if run_local('which rsync').present? && $?.success?
          execute_on_servers do |servers|
            commands = servers.map do |server|
              target = sessions[server]
              user = target.options[:user] || fetch(:user, nil)
              host = target.host
              port = target.options[:port]
              full_host = "#{"#{user}@" if user.present?}#{host}"

              ssh = port.present? ? "ssh -p #{port}" : 'ssh'
              cmd = %W[rsync -P -e #{ssh} --timeout=15]
              case direction
              when :up
                cmd << from << "#{full_host}:#{to}"
              when :down
                cmd << "#{full_host}:#{from}" << to
              else
                raise "Don't know how to transfer in direction #{direction}"
              end
              cmd.shelljoin
            end
            commands.each do |cmd|
              logger.info cmd if logger

              3.times do
                break if system(cmd)
                break unless [10, 11, 12, 23, 30, 35].include?($?.exitstatus)
              end
              raise "rsync returned #{$?.exitstatus}" unless $?.success?
            end
          end
        end
      when :sftp, :scp
        DumpRake::ContiniousTimeout.timeout 15 do |thread|
          transfer(direction, from, to, :via => via) do |channel, path, transfered, total|
            thread.defer
            progress = if transfered < total
              "\e[1m%5.1f%%\e[0m" % (transfered * 100.0 / total)
            else
              '100%'
            end
            $stderr << "\rTransfering: #{progress}"
          end
        end
      else
        raise "Unknown transfer method #{via}"
      end
    end

    def do_transfer(direction, from, to)
      if via = DumpRake::Env[:transfer_via]
        case via.downcase
        when 'rsync'
          do_transfer_via(:rsync, direction, from, to)
        when 'sftp'
          do_transfer_via(:sftp, direction, from, to)
        when 'scp'
          do_transfer_via(:scp, direction, from, to)
        else
          raise "Unknown transfer method #{via}"<commit_msg>use $CHILD_STATUS instead of $?<commit_after># encoding: UTF-8

require 'fileutils'
require 'shellwords'

require 'dump_rake/continious_timeout'
require 'dump_rake/env'

require 'active_support/core_ext/object/blank'

require 'English'

Capistrano::Configuration.instance(:i_need_this!).load do
  namespace :dump do
    def dump_command(command, env = {})
      rake = env.delete(:rake) || 'rake'

      # stringify_keys! from activesupport
      DumpRake::Env.stringify!(env)

      env.update(DumpRake::Env.for_command(command, true))

      cmd = %W[-s dump:#{command}]
      cmd += env.sort.map{ |key, value| "#{key}=#{value}" }
      "#{rake} #{cmd.shelljoin}"
    end

    def fetch_rails_env
      fetch(:rails_env, 'production')
    end

    def got_rsync?
      `which rsync`
      $CHILD_STATUS.success?
    end

    def do_transfer_via(via, direction, from, to)
      case via
      when :rsync
        if run_local('which rsync').present? && $CHILD_STATUS.success?
          execute_on_servers do |servers|
            commands = servers.map do |server|
              target = sessions[server]
              user = target.options[:user] || fetch(:user, nil)
              host = target.host
              port = target.options[:port]
              full_host = "#{"#{user}@" if user.present?}#{host}"

              ssh = port.present? ? "ssh -p #{port}" : 'ssh'
              cmd = %W[rsync -P -e #{ssh} --timeout=15]
              case direction
              when :up
                cmd << from << "#{full_host}:#{to}"
              when :down
                cmd << "#{full_host}:#{from}" << to
              else
                raise "Don't know how to transfer in direction #{direction}"
              end
              cmd.shelljoin
            end
            commands.each do |cmd|
              logger.info cmd if logger

              3.times do
                break if system(cmd)
                break unless [10, 11, 12, 23, 30, 35].include?($CHILD_STATUS.exitstatus)
              end
              raise "rsync returned #{$CHILD_STATUS.exitstatus}" unless $CHILD_STATUS.success?
            end
          end
        end
      when :sftp, :scp
        DumpRake::ContiniousTimeout.timeout 15 do |thread|
          transfer(direction, from, to, :via => via) do |channel, path, transfered, total|
            thread.defer
            progress = if transfered < total
              "\e[1m%5.1f%%\e[0m" % (transfered * 100.0 / total)
            else
              '100%'
            end
            $stderr << "\rTransfering: #{progress}"
          end
        end
      else
        raise "Unknown transfer method #{via}"
      end
    end

    def do_transfer(direction, from, to)
      if via = DumpRake::Env[:transfer_via]
        case via.downcase
        when 'rsync'
          do_transfer_via(:rsync, direction, from, to)
        when 'sftp'
          do_transfer_via(:sftp, direction, from, to)
        when 'scp'
          do_transfer_via(:scp, direction, from, to)
        else
          raise "Unknown transfer method #{via}"
{'fraction_non_alphanumeric': 0.1206896551724138, 'fraction_numerical': 0.009615384615384616, 'mean_word_length': 1.8165266106442577, 'pattern_counts': {'":': 0, '<': 25, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5906140', 'n_tokens_mistral': 1842, 'n_tokens_neox': 1767, 'n_words': 461}