<commit_before>require 'awesome_spawn'
require 'pathname'
require 'linux_admin'

RAILS_ROOT ||= Pathname.new(__dir__).join("../../../")

class PostgresAdmin
  def self.pg_ctl
    Pathname.new(ENV.fetch("APPLIANCE_PG_CTL"))
  end

  def self.data_directory
    Pathname.new(ENV.fetch("APPLIANCE_PG_DATA"))
  end

  def self.mount_point
    Pathname.new(ENV.fetch("APPLIANCE_PG_MOUNT_POINT"))
  end

  def self.template_directory
    Pathname.new(ENV.fetch("APPLIANCE_TEMPLATE_DIRECTORY"))
  end

  def self.service_name
    ENV.fetch("APPLIANCE_PG_SERVICE")
  end

  def self.scl_name
    ENV.fetch('APPLIANCE_PG_SCL_NAME')
  end

  def self.scl_enable_prefix
    "scl enable #{scl_name}"
  end

  def self.package_name
    ENV.fetch('APPLIANCE_PG_PACKAGE_NAME')
  end

  # Unprivileged user to run postgresql
  def self.user
    "postgres".freeze
  end

  def self.certificate_location
    RAILS_ROOT.join("certs")
  end

  def self.logical_volume_name
    "lv_pg".freeze
  end

  def self.volume_group_name
    "vg_data".freeze
  end

  def self.database_disk_filesystem
    "xfs".freeze
  end

  def self.initialized?
    !Dir[data_directory.join("*")].empty?
  end

  def self.service_running?
    LinuxAdmin::Service.new(service_name).running?
  end

  def self.local_server_in_recovery?
    data_directory.join("recovery.conf").exist?
  end

  def self.local_server_status
    if service_running?
      "running (#{local_server_in_recovery? ? "standby" : "primary"})"
    elsif initialized?
      "initialized and stopped"
    else
      "not initialized"
    end
  end

  def self.logical_volume_path
    Pathname.new("/dev").join(volume_group_name, logical_volume_name)
  end

  def self.database_size(opts)
    result = runcmd("psql", opts, :command => "SELECT pg_database_size('#{opts[:dbname]}');")
    result.match(/^\s+([0-9]+)\n/)[1].to_i
  end

  def self.prep_data_directory
    # initdb will fail if the database directory is not empty or not owned by the PostgresAdmin.user
    FileUtils.mkdir(PostgresAdmin.data_directory) unless Dir.exist?(PostgresAdmin.data_directory)
    FileUtils.chown_R(PostgresAdmin.user, PostgresAdmin.user, PostgresAdmin.data_directory)
    FileUtils.rm_rf(PostgresAdmin.data_directory.children.map(&:to_s))
  end

  def self.backup(opts)
    before_backup(opts)
    backup_pg_compress(opts)
  end

  def self.before_backup(_opts)
    # TODO: check disk space
  end

  def self.restore(opts)
    before_restore(opts)
    restore_pg_compress(opts)
  end

  def self.before_restore(opts)
    runcmd("psql", opts, :command => <<-SQL)
      SELECT
        drop_subscription
      FROM
        pglogical.subscription subs,
        LATERAL pglogical.drop_subscription(subs.sub_name)
    SQL
    runcmd("psql", opts, :command => <<-SQL)
      DROP EXTENSION pglogical CASCADE
    SQL
  rescue AwesomeSpawn::CommandResultError
    $log.info("MIQ(#{name}.#{__method__}) Ignoring failure to remove pglogical before restore ...")
  end

  def self.backup_compress_with_gzip
    # 1) Use compressed dumps.  You can use your favorite compression program, for example gzip:
    #
    # pg_dump dbname | gzip > filename.gz
  end

  def self.restore_compress_with_zip
    # Reload with:
    #
    # gunzip -c filename.gz | psql dbname
    # or:
    #
    # cat filename.gz | gunzip | psql dbname
  end

  def self.backup_split_by_size
    # 2) Use split. The split command allows you to split the output into pieces that are acceptable in size to the underlying file system. For example, to make chunks of 1 megabyte:
    #
    # pg_dump dbname | split -b 1m - filename
  end

  def self.restore_split_by_size
    # Reload with:
    #
    # cat filename* | psql dbname
  end
<commit_msg>Move logic for dropping pglogical extension to a separate method<commit_after>require 'awesome_spawn'
require 'pathname'
require 'linux_admin'

RAILS_ROOT ||= Pathname.new(__dir__).join("../../../")

class PostgresAdmin
  def self.pg_ctl
    Pathname.new(ENV.fetch("APPLIANCE_PG_CTL"))
  end

  def self.data_directory
    Pathname.new(ENV.fetch("APPLIANCE_PG_DATA"))
  end

  def self.mount_point
    Pathname.new(ENV.fetch("APPLIANCE_PG_MOUNT_POINT"))
  end

  def self.template_directory
    Pathname.new(ENV.fetch("APPLIANCE_TEMPLATE_DIRECTORY"))
  end

  def self.service_name
    ENV.fetch("APPLIANCE_PG_SERVICE")
  end

  def self.scl_name
    ENV.fetch('APPLIANCE_PG_SCL_NAME')
  end

  def self.scl_enable_prefix
    "scl enable #{scl_name}"
  end

  def self.package_name
    ENV.fetch('APPLIANCE_PG_PACKAGE_NAME')
  end

  # Unprivileged user to run postgresql
  def self.user
    "postgres".freeze
  end

  def self.certificate_location
    RAILS_ROOT.join("certs")
  end

  def self.logical_volume_name
    "lv_pg".freeze
  end

  def self.volume_group_name
    "vg_data".freeze
  end

  def self.database_disk_filesystem
    "xfs".freeze
  end

  def self.initialized?
    !Dir[data_directory.join("*")].empty?
  end

  def self.service_running?
    LinuxAdmin::Service.new(service_name).running?
  end

  def self.local_server_in_recovery?
    data_directory.join("recovery.conf").exist?
  end

  def self.local_server_status
    if service_running?
      "running (#{local_server_in_recovery? ? "standby" : "primary"})"
    elsif initialized?
      "initialized and stopped"
    else
      "not initialized"
    end
  end

  def self.logical_volume_path
    Pathname.new("/dev").join(volume_group_name, logical_volume_name)
  end

  def self.database_size(opts)
    result = runcmd("psql", opts, :command => "SELECT pg_database_size('#{opts[:dbname]}');")
    result.match(/^\s+([0-9]+)\n/)[1].to_i
  end

  def self.prep_data_directory
    # initdb will fail if the database directory is not empty or not owned by the PostgresAdmin.user
    FileUtils.mkdir(PostgresAdmin.data_directory) unless Dir.exist?(PostgresAdmin.data_directory)
    FileUtils.chown_R(PostgresAdmin.user, PostgresAdmin.user, PostgresAdmin.data_directory)
    FileUtils.rm_rf(PostgresAdmin.data_directory.children.map(&:to_s))
  end

  def self.backup(opts)
    before_backup(opts)
    backup_pg_compress(opts)
  end

  def self.before_backup(_opts)
    # TODO: check disk space
  end

  def self.restore(opts)
    before_restore(opts)
    restore_pg_compress(opts)
  end

  def self.before_restore(opts)
    # Drop subscriptions, unload extension and ensure pglogical connections are closed before proceeding
    unload_pglogical_extension(opts)
  rescue AwesomeSpawn::CommandResultError
    $log.info("MIQ(#{name}.#{__method__}) Ignoring failure to remove pglogical before restore ...")
  end

  def self.unload_pglogical_extension(opts)
    runcmd("psql", opts, :command => <<-SQL)
      SELECT
        drop_subscription
      FROM
        pglogical.subscription subs,
        LATERAL pglogical.drop_subscription(subs.sub_name)
    SQL

    runcmd("psql", opts, :command => <<-SQL)
      DROP EXTENSION pglogical CASCADE
    SQL
  end

  def self.backup_compress_with_gzip
    # 1) Use compressed dumps.  You can use your favorite compression program, for example gzip:
    #
    # pg_dump dbname | gzip > filename.gz
  end

  def self.restore_compress_with_zip
    # Reload with:
    #
    # gunzip -c filename.gz | psql dbname
    # or:
    #
    # cat filename.gz | gunzip | psql dbname
  end

  def self.backup_split_by_size
    # 2) Use split. The split command allows you to split the output into pieces that are acceptable in size to the underlying file system. For example, to make chunks of 1 megabyte:
    #
    # pg_dump dbname | split -b 1m - filename
  end

  def self.restore_split_by_size
    # Reload with:
    #
    # cat filename* | psql dbname
  end

{'fraction_non_alphanumeric': 0.0965472312703583, 'fraction_numerical': 0.0018241042345276872, 'mean_word_length': 3.509988249118684, 'pattern_counts': {'":': 0, '<': 11, '<?xml version=': 0, '>': 11, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '353957', 'n_tokens_mistral': 2685, 'n_tokens_neox': 2613, 'n_words': 581}