<commit_before>require 'cl'
require 'fileutils'
require 'forwardable'
require 'shellwords'
require 'dpl/helper/assets'
require 'dpl/helper/cmd'
require 'dpl/helper/env'
require 'dpl/helper/interpolate'
require 'dpl/helper/squiggle'
require 'dpl/provider/dsl'
require 'dpl/provider/examples'
require 'dpl/version'

module Dpl
  # Base class for all concrete providers that `dpl` supports.
  #
  # These are subclasses of `Cl::Cmd` which means they are going to be detected
  # by the first argument passed to `dpl [provider]`, instantiated, and run.
  #
  # Implementors are encouraged to use the provider DSL to declare various
  # features, requirements, and attributes that apply to their provider, to
  # implement any of the following stages (methods) according to their needs
  # and semantics:
  #
  #   * init
  #   * install
  #   * login
  #   * setup
  #   * validate
  #   * prepare
  #   * deploy
  #   * finish
  #
  # The main logic should sit in the `deploy` stage.
  #
  # If at any time the method `error` is called, or any exception raised the
  # deploy process will be halted, and subsequent stages skipped. However, the
  # stage `finish` will run even if previous stages have raised an error,
  # giving the provider the opportunity to potentially clean up stage.
  #
  # In addition to this the following methods will be called if implemented
  # by the provider:
  #
  #   * run_cmd
  #   * add_key
  #   * remove_key
  #
  # Like the `finish` stage, the method `remove_key` will be called even if
  # previous stages have raised an error.
  #
  # See the respective method's documentation for details on these.
  #
  # The following stages are not meant to be overwritten, but considered
  # internal:
  #
  #   * before_install
  #   * before_setup
  #   * before_prepare
  #   * before_finish
  #
  # Dependencies declared as required, such as APT, NPM, or Python are going to
  # be installed as part of the `before_install` stage .
  #
  # Cleanup is run as part of the `before_prepare` stage if the option
  # `--cleanup` was given. This will use `git stash --all` in order to reset
  # the working directory to the committed state, and cleanup any left over
  # artifacts from the build process. Providers can use the DSL method `keep`
  # in order to declare known artifacts (such as CLI tooling installed to the
  # working directory) that needs to be moved out of the way and restored after
  # the cleanup process. (It is recommended to place such artifacts outside of
  # the build working directory though, for example in `~/.dpl`).
  #
  # The method `run_cmd` is called for each command specified using the `--run`
  # option. By default, these command are going to be run as local shell
  # commands, but providers can choose to overwrite this method in order to run
  # the command on a remote machine.
  #
  # @see https://github.com/svenfuchs/cl Cl's documentation for details on how
  # providers (commands) are declared and run.

  class Provider < Cl::Cmd
    extend Dsl, Forwardable
    include Assets, Env, FileUtils, Interpolate, Squiggle

    class << self
      def examples
        @examples ||= super || Examples.new(self).cmds
      end

      def move_files(ctx)
        ctx.move_files(move) if move.any?
      end

      def unmove_files(ctx)
        ctx.unmove_files(move) if move.any?
      end

      def install_deps?
        apt? || gem? || npm? || pip?
      end

      def install_deps(ctx)
        ctx.apts_get(apt) if apt?
        ctx.gems_require(gem) if gem?
        npm.each { |npm| ctx.npm_install *npm } if npm?
        pip.each { |pip| ctx.pip_install *pip } if pip?
      end

      def validate_runtimes(ctx)
        ctx.validate_runtimes(runtimes) if runtimes.any?
      end
    end

    # Fold names to display in the build log.
    FOLDS = {
      init:     'Initialize deployment',
      setup:    'Setup deployment',
      validate: 'Validate deployment',
      install:  'Install deployment dependencies',
      login:    'Authenticate deployment',
      prepare:  'Prepare deployment',
      deploy:   'Run deployment',
      finish:   'Finish deployment',
    }

    # Deployment process stages.
    #
    # In addition to the stages listed here the stage `finish` will be run at
    # the end of the process.
    #
    # Also, the methods `add_key` (called before `setup`), `remove_key` (called
    # before `finish`), and `run_cmd` (called after `deploy`) may be of
    # interest to implementors.
    STAGES = %i(
      init
      install
      login
      setup
      validate
      prepare
      deploy
    )

    abstract

    arg :provider, 'The provider name', required: true

    opt '--run CMD',      'Command to execute after the deployment finished successfully', type: :array
    opt '--cleanup',      'Skip cleaning up build artifacts before the deployment', negate: %w(skip)
    opt '--stage NAME',   'Execute the given stage(s) only', type: :array, internal: true, default: STAGES
    opt '--backtrace',    'Print the backtrace for exceptions', internal: true
    opt '--fold',         'Wrap log output in folds', internal: true

    msgs before_install:  'Installing deployment dependencies',
         before_setup:    'Setting the build environment up for the deployment',
         setup_git_ssh:   'Setting up git-ssh',
         cleanup:         'Cleaning up git repository with `git stash --all`',
         ssh_keygen:      'Generating SSH key',
         setup_git_ua:    'Setting up git HTTP user agent',
         ssh_remote_host: 'SSH remote is %s at port %s',
         ssh_try_connect: 'Waiting for SSH connection ...',
         ssh_connected:   'SSH connection established.',
         ssh_failed:      'Failed to establish SSH connection.'

    def_delegators :'self.class', :status, :full_name, :install_deps,
      :install_deps?, :keep, :move_files, :unmove_files, :needs?, :runtimes,
      :validate_runtimes, :user_agent

    def_delegators :ctx, :apt_get, :gem_require, :npm_install, :pip_install,
      :build_dir, :build_number, :repo_slug, :encoding, :git_commit_msg,
      :git_dirty?, :git_log, :git_ls_files, :git_ls_remote?, :git_remote_urls,
      :git_rev_parse, :git_sha, :git_tag, :machine_name, :node_version,
      :npm_version, :sleep, :ssh_keygen, :success?, :mv, :tmp_dir, :which,
      :logger, :rendezvous, :file_size, :write_file, :write_netrc, :last_out,
      :last_err, :tty?

    attr_reader :repo_name, :key_name

    def initialize(ctx, *args)
      @repo_name = ctx.repo_name
      @key_name = ctx.machine_name
      super
    end

    # Runs all stages, all commands provided by the user, as well as the final
    # stage `finish` (which will be run even if an error has been raised during
    # previous stages).
    def run
      stages = stage.select { |stage| run_stage?(stage) }
      stages.each { |stage| run_stage(stage) }
      run_cmds
    rescue Error
      raise
    rescue Exception => e
      raise Error.new("#{e.message} (#{e.class})", backtrace: backtrace? ? caller : nil)
    ensure
      run_stage(:finish, fold: false) if finish?
    end

    # Whether or not a stage needs to be run
    def run_stage?(stage)
      respond_to?(:"before_#{stage}") || respond_to?(stage)
    end

    def finish?
      stage.size == STAGES.size
    end<commit_msg>during tests just re-raise<commit_after>require 'cl'
require 'fileutils'
require 'forwardable'
require 'shellwords'
require 'dpl/helper/assets'
require 'dpl/helper/cmd'
require 'dpl/helper/env'
require 'dpl/helper/interpolate'
require 'dpl/helper/squiggle'
require 'dpl/provider/dsl'
require 'dpl/provider/examples'
require 'dpl/version'

module Dpl
  # Base class for all concrete providers that `dpl` supports.
  #
  # These are subclasses of `Cl::Cmd` which means they are going to be detected
  # by the first argument passed to `dpl [provider]`, instantiated, and run.
  #
  # Implementors are encouraged to use the provider DSL to declare various
  # features, requirements, and attributes that apply to their provider, to
  # implement any of the following stages (methods) according to their needs
  # and semantics:
  #
  #   * init
  #   * install
  #   * login
  #   * setup
  #   * validate
  #   * prepare
  #   * deploy
  #   * finish
  #
  # The main logic should sit in the `deploy` stage.
  #
  # If at any time the method `error` is called, or any exception raised the
  # deploy process will be halted, and subsequent stages skipped. However, the
  # stage `finish` will run even if previous stages have raised an error,
  # giving the provider the opportunity to potentially clean up stage.
  #
  # In addition to this the following methods will be called if implemented
  # by the provider:
  #
  #   * run_cmd
  #   * add_key
  #   * remove_key
  #
  # Like the `finish` stage, the method `remove_key` will be called even if
  # previous stages have raised an error.
  #
  # See the respective method's documentation for details on these.
  #
  # The following stages are not meant to be overwritten, but considered
  # internal:
  #
  #   * before_install
  #   * before_setup
  #   * before_prepare
  #   * before_finish
  #
  # Dependencies declared as required, such as APT, NPM, or Python are going to
  # be installed as part of the `before_install` stage .
  #
  # Cleanup is run as part of the `before_prepare` stage if the option
  # `--cleanup` was given. This will use `git stash --all` in order to reset
  # the working directory to the committed state, and cleanup any left over
  # artifacts from the build process. Providers can use the DSL method `keep`
  # in order to declare known artifacts (such as CLI tooling installed to the
  # working directory) that needs to be moved out of the way and restored after
  # the cleanup process. (It is recommended to place such artifacts outside of
  # the build working directory though, for example in `~/.dpl`).
  #
  # The method `run_cmd` is called for each command specified using the `--run`
  # option. By default, these command are going to be run as local shell
  # commands, but providers can choose to overwrite this method in order to run
  # the command on a remote machine.
  #
  # @see https://github.com/svenfuchs/cl Cl's documentation for details on how
  # providers (commands) are declared and run.

  class Provider < Cl::Cmd
    extend Dsl, Forwardable
    include Assets, Env, FileUtils, Interpolate, Squiggle

    class << self
      def examples
        @examples ||= super || Examples.new(self).cmds
      end

      def move_files(ctx)
        ctx.move_files(move) if move.any?
      end

      def unmove_files(ctx)
        ctx.unmove_files(move) if move.any?
      end

      def install_deps?
        apt? || gem? || npm? || pip?
      end

      def install_deps(ctx)
        ctx.apts_get(apt) if apt?
        ctx.gems_require(gem) if gem?
        npm.each { |npm| ctx.npm_install *npm } if npm?
        pip.each { |pip| ctx.pip_install *pip } if pip?
      end

      def validate_runtimes(ctx)
        ctx.validate_runtimes(runtimes) if runtimes.any?
      end
    end

    # Fold names to display in the build log.
    FOLDS = {
      init:     'Initialize deployment',
      setup:    'Setup deployment',
      validate: 'Validate deployment',
      install:  'Install deployment dependencies',
      login:    'Authenticate deployment',
      prepare:  'Prepare deployment',
      deploy:   'Run deployment',
      finish:   'Finish deployment',
    }

    # Deployment process stages.
    #
    # In addition to the stages listed here the stage `finish` will be run at
    # the end of the process.
    #
    # Also, the methods `add_key` (called before `setup`), `remove_key` (called
    # before `finish`), and `run_cmd` (called after `deploy`) may be of
    # interest to implementors.
    STAGES = %i(
      init
      install
      login
      setup
      validate
      prepare
      deploy
    )

    abstract

    arg :provider, 'The provider name', required: true

    opt '--run CMD',      'Command to execute after the deployment finished successfully', type: :array
    opt '--cleanup',      'Skip cleaning up build artifacts before the deployment', negate: %w(skip)
    opt '--stage NAME',   'Execute the given stage(s) only', type: :array, internal: true, default: STAGES
    opt '--backtrace',    'Print the backtrace for exceptions', internal: true
    opt '--fold',         'Wrap log output in folds', internal: true

    msgs before_install:  'Installing deployment dependencies',
         before_setup:    'Setting the build environment up for the deployment',
         setup_git_ssh:   'Setting up git-ssh',
         cleanup:         'Cleaning up git repository with `git stash --all`',
         ssh_keygen:      'Generating SSH key',
         setup_git_ua:    'Setting up git HTTP user agent',
         ssh_remote_host: 'SSH remote is %s at port %s',
         ssh_try_connect: 'Waiting for SSH connection ...',
         ssh_connected:   'SSH connection established.',
         ssh_failed:      'Failed to establish SSH connection.'

    def_delegators :'self.class', :status, :full_name, :install_deps,
      :install_deps?, :keep, :move_files, :unmove_files, :needs?, :runtimes,
      :validate_runtimes, :user_agent

    def_delegators :ctx, :apt_get, :gem_require, :npm_install, :pip_install,
      :build_dir, :build_number, :repo_slug, :encoding, :git_commit_msg,
      :git_dirty?, :git_log, :git_ls_files, :git_ls_remote?, :git_remote_urls,
      :git_rev_parse, :git_sha, :git_tag, :machine_name, :node_version,
      :npm_version, :sleep, :ssh_keygen, :success?, :mv, :tmp_dir, :which,
      :logger, :rendezvous, :file_size, :write_file, :write_netrc, :last_out,
      :last_err, :test?, :tty?

    attr_reader :repo_name, :key_name

    def initialize(ctx, *args)
      @repo_name = ctx.repo_name
      @key_name = ctx.machine_name
      super
    end

    # Runs all stages, all commands provided by the user, as well as the final
    # stage `finish` (which will be run even if an error has been raised during
    # previous stages).
    def run
      stages = stage.select { |stage| run_stage?(stage) }
      stages.each { |stage| run_stage(stage) }
      run_cmds
    rescue Error
      raise
    rescue Exception => e
      raise Error.new("#{e.message} (#{e.class})", backtrace: backtrace? ? caller : nil) unless test?
      raise
    ensure
      run_stage(:finish, fold: false) if finish?
    end

    # Whether or not a stage needs to be run
    def run_stage?(stage)
      respond_to?(:"before_#{stage}") || respond_to?(stage)
    end

    def finish?
      stage.size == STAGES.size
    end
{'fraction_non_alphanumeric': 0.0927391838405896, 'fraction_numerical': 0.0, 'mean_word_length': 2.8862370723945903, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 5, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2714655', 'n_tokens_mistral': 4486, 'n_tokens_neox': 4360, 'n_words': 1634}