<commit_before># encoding: utf-8

module Backup
  module CLI

    ##
    # Wrapper method for %x[] to run CL commands
    # through a ruby method. This helps with test coverage and
    # improves readability.
    #
    # It'll first remove all prefixing slashes ( / ) by using .gsub(/^\s+/, '')
    # This allows for the EOS blocks to be indented without actually using any
    # prefixing spaces. This cleans up the implementation code.
    #
    # Every time the Backup::CLI#run method is invoked, it'll invoke
    # the Backup::CLI#raise_if_command_not_found method after running the
    # requested command on the OS.
    #
    # Backup::CLI#raise_if_command_not_found takes a single argument, the utility name.
    # the command.slice(0, command.index(/\s/)).split('/')[-1] line will extract only the utility
    # name (e.g. mongodump, pgdump, etc) from a command like "/usr/local/bin/mongodump <options>"
    # and pass that in to the Backup::CLI#raise_if_command_not_found
    def run(command)
      command.gsub!(/^\s+/, '')
      raise_if_command_not_found!(
        command_name(command)
      )

      pid, stdin, stdout, stderr = Open4::popen4(command)
      ignored, @status = Process::waitpid2(pid)
      @stdout = stdout.read
      @stderr = stderr.read

      raise_if_command_failed!(command_name(command), @status)
    end

    ##
    # Wrapper method for FileUtils.mkdir_p to create directories
    # through a ruby method. This helps with test coverage and
    # improves readability
    def mkdir(path)
      FileUtils.mkdir_p(path)
    end

    ##
    # Wrapper for the FileUtils.rm_rf to remove files and folders
    # through a ruby method. This helps with test coverage and
    # improves readability
    def rm(path)
      FileUtils.rm_rf(path)
    end

    ##
    # Tries to find the full path of the specified utility. If the full
    # path is found, it'll return that. Otherwise it'll just return the
    # name of the utility. If the 'utility_path' is defined, it'll check
    # to see if it isn't an empty string, and if it isn't, it'll go ahead and
    # always use that path rather than auto-detecting it
    def utility(name)
      if respond_to?(:utility_path)
        if utility_path.is_a?(String) and not utility_path.empty?
          return utility_path
        end
      end

      if path = %x[which #{name}].chomp and not path.empty?
        return path
      end
      name
    end

    def command_name(command)
      command.slice(0, command.index(/\s/)).split('/')[-1]
    end

    ##
    # If the command that was previously run via this Ruby process returned
    # error code "32512", the invoked utility (e.g. mysqldump, pgdump, etc) could not be found.
    # If this is the case then this method will throw an exception, informing the user of this problem.
    #
    # Since this raises an exception, it'll stop the entire backup process, clean up the temp files
    # and notify the user via the built-in notifiers if these are set.
    def raise_if_command_not_found!(utility)
      if $?.to_i.eql?(32512)
        raise Exception::CommandNotFound , "Could not find the utility \"#{utility}\" on \"#{RUBY_PLATFORM}\".\n" +
                                           "If this is a database utility, try defining the 'utility_path' option in the configuration file.\n" +
                                           "See the Database Wiki for more information about the Utility Path option."
      end
    end

    ##
    # If the command that was previously run via this Ruby process returned
    # a non-zero error code, the invoked utility (e.g. mysqldump, pgdump, etc) failed to run.
    # If this is the case then this method will throw an exception, informing the user of this problem.
    #
    # Since this raises an exception, it'll stop the entire backup process, clean up the temp files
    # and notify the user via the built-in notifiers if these are set.
    def raise_if_command_failed!(utility, status)
      unless status.to_i.eql?(0)
        raise Exception::CommandFailed , "Failed to run \"#{utility}\" on \"#{RUBY_PLATFORM}\".\n" +
                                         "The status code returned was #{status}\n" +
                                         "STDOUT was:\n" +
                                         @stdout +
                                         "STDERR was:\n" +
                                         @stderr
      end
    end

  end
end<commit_msg>Revamped the Backup::CLI module. Removed the CommandNotFound exception and method since the CommandFailed exception already picks it up, and it isn't necessary to check whether the command isn't found twice. Also, it is now possible to pass in an array of exit codes to ignore in to the #run method, rather than raising an exception when a non-0 exit code gets returned. Cleaned up the rest of the code in the cli.rb file.<commit_after># encoding: utf-8

module Backup
  module CLI

    ##
    # Runs a given command in an isolated (sub) process using POpen4.
    # The STDOUT, STDERR and the returned exit code of the utility will be stored in the process_data Hash.
    #
    # If a command returns an exit code other than 0, an exception will raise and the backup process will abort.
    # Some utilities return exit codes other than 0 which aren't an issue in Backup's context. If this is the case,
    # you can pass in an array of exit codes to ignore (whitelist), for example:
    #
    #   run("tar -cf /output.tar /some/folder", :ignore_exit_codes => [1])
    #
    # So if the `tar` utility returns in this case 1, Backup will consider it an acceptable return code.
    #
    # Note: Exit code 0 is always automatically added to the :ignore_exit_codes array, regardless of whether you specify an
    #       array to ignore or not.
    def run(command, options = {})
      command.gsub!(/^\s+/, "")

      process_data                     = Hash.new
      pid, stdin, stdout, stderr       = Open4::popen4(command)
      ignored, process_data[:status]   = Process::waitpid2(pid)
      process_data[:stdout]            = stdout.read
      process_data[:stderr]            = stderr.read
      process_data[:ignore_exit_codes] = ((options[:ignore_exit_codes] || Array.new) << 0).uniq

      raise_if_command_failed!(command_name(command), process_data)
    end

    ##
    # Wrapper method for FileUtils.mkdir_p to create directories
    # through a ruby method. This helps with test coverage and
    # improves readability
    def mkdir(path)
      FileUtils.mkdir_p(path)
    end

    ##
    # Wrapper for the FileUtils.rm_rf to remove files and folders
    # through a ruby method. This helps with test coverage and
    # improves readability
    def rm(path)
      FileUtils.rm_rf(path)
    end

    ##
    # Tries to find the full path of the specified utility. If the full
    # path is found, it'll return that. Otherwise it'll just return the
    # name of the utility. If the 'utility_path' is defined, it'll check
    # to see if it isn't an empty string, and if it isn't, it'll go ahead and
    # always use that path rather than auto-detecting it
    def utility(name)
      if respond_to?(:utility_path)
        if utility_path.is_a?(String) and not utility_path.empty?
          return utility_path
        end
      end

      if path = %x[which #{name}].chomp and not path.empty?
        return path
      end
      name
    end

    ##
    # Returns the name of the command
    def command_name(command)
      command.slice(0, command.index(/\s/)).split('/')[-1]
    end

    ##
    # Inspects the exit code returned from the POpen4 child process. If the exit code isn't listed
    # in the process_data[:ignore_exit_codes] array, an exception will be raised, aborting the backup process.
    #
    # Information regarding the error ( EXIT CODE and STDERR ) will be returned to the shell so the user can
    # investigate the issue.
    #
    # The "Exception::CommandFailed" exception will be raised.
    def raise_if_command_failed!(utility, process_data)
      unless process_data[:ignore_exit_codes].include?(process_data[:status].to_i)
        exception_string =  "Failed to run \"#{ utility }\" on \"#{ RUBY_PLATFORM }\".\n\n" +
                            "Exit code was:\n\n\s\s#{ process_data[:status].to_i }\n\n"
        exception_string << "STDOUT was:\n\n\s\s#{ process_data[:stdout].gsub("\n", "\n\s\s") }" unless process_data[:stdout].empty?
        exception_string << "STDERR was:\n\n\s\s#{ process_data[:stderr].gsub("\n", "\n\s\s") }" unless process_data[:stderr].empty?

        raise Exception::CommandFailed, exception_string
      end
    end

  end
end
{'fraction_non_alphanumeric': 0.08746962860118014, 'fraction_numerical': 0.003933819275714451, 'mean_word_length': 2.5152501016673443, 'pattern_counts': {'":': 0, '<': 10, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 14, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6768399', 'n_tokens_mistral': 2508, 'n_tokens_neox': 2431, 'n_words': 956}