<commit_before># @package MiGA
# @license Artistic-2.0

require 'miga/cli/action/doctor/base'

class MiGA::Cli::Action::Doctor < MiGA::Cli::Action
  include MiGA::Cli::Action::Doctor::Base

  def parse_cli
    cli.defaults = { threads: 1 }
    cli.defaults = Hash[@@OPERATIONS.keys.map { |i| [i, true] }]
    cli.parse do |opt|
      operation_n = Hash[@@OPERATIONS.map { |k, v| [v[0], k] }]
      cli.opt_object(opt, [:project])
      opt.on(
        '--ignore TASK1,TASK2', Array,
        'Do not perform the task(s) listed. Available tasks are:',
        * @@OPERATIONS.values.map { |v| "~ #{v[0]}: #{v[1]}" }
      ) { |v| v.map { |i| cli[operation_n[i]] = false } }
      opt.on(
        '--only TASK',
        'Perform only the specified task (see --ignore)'
      ) do |v|
        op_k = @@OPERATIONS.find { |_, i| i[0] == v.downcase }.first
        @@OPERATIONS.each_key { |i| cli[i] = false }
        cli[op_k] = true
      end
      opt.on(
        '-t', '--threads INT', Integer,
        "Concurrent threads to use. By default: #{cli[:threads]}"
      ) { |v| cli[:threads] = v }
    end
  end

  def perform
    p = cli.load_project
    @@OPERATIONS.keys.each do |k|
      send("check_#{k}", cli) if cli[k]
    end
  end

  @@OPERATIONS = {
    status: ['status', 'Update metadata status of all datasets'],
    db: ['databases', 'Check integrity of database files'],
    bidir: ['bidirectional', 'Check distances are bidirectional'],
    dist: ['distances', 'Check distance summary tables'],
    files: ['files', 'Check for outdated files'],
    cds: ['cds', 'Check for gzipped genes and proteins'],
    ess: ['essential-genes', 'Check for unarchived essential genes'],
    mts: ['mytaxa-scan', 'Check for unarchived MyTaxa scan'],
    start: ['start', 'Check for lingering .start files'],
    tax: ['taxonomy', 'Check for taxonomy consistency (not yet implemented)']
  }

  class << self
    ##
    # All supported operations
    def OPERATIONS
      @@OPERATIONS
    end
  end

  ##
  # Perform status operation with MiGA::Cli +cli+
  def check_status(cli)
    cli.say 'Updating metadata status'
    p = cli.load_project
    n = p.dataset_names.size
    (0 .. cli[:threads] - 1).map do |i|
      Process.fork do
        k = 0
        cli.load_project.each_dataset do |d|
          k += 1
          cli.advance('Datasets:', k, n, false) if i == 0
          d.recalculate_status if k % cli[:threads] == i
        end
      end
    end
    Process.waitall
    cli.say
  end

  ##
  # Perform databases operation with MiGA::Cli +cli+
  def check_db(cli)
    cli.say 'Checking integrity of databases'
    p = cli.load_project
    n = p.dataset_names.size
    (0 .. cli[:threads] - 1).map do |i|
      Process.fork do
        k = 0
        p.each_dataset do |d|
          k += 1
          cli.advance('Datasets:', k, n, false) if i == 0
          next unless k % cli[:threads] == i
          each_database_file(d) do |db_file, metric, result, _rank|
            check_sqlite3_database(db_file, metric) do
              cli.say(
                "  > Removing malformed database from #{d.name}:#{result}   "
              )
              File.unlink(db_file)
              r = d.result(result) or next
              [r.path(:done), r.path].each do |f|
                File.unlink(f) if File.exist?(f)
              end
            end
          end
        end
      end
    end
    Process.waitall
    cli.say
  end

  ##
  # Perform bidirectional operation with MiGA::Cli +cli+
  def check_bidir(cli)
    cli.say 'Checking if reference distances are bidirectional'
    project = cli.load_project
    ref_ds = project.each_dataset.select(&:ref?)
    ref_names = ref_ds.map(&:name)
    n = ref_ds.size

    # Read data first (threaded)
    tmp = File.join(project.path, 'doctor-bidirectional.tmp')
    FileUtils.mkdir_p(tmp)
    MiGA::Parallel.process(cli[:threads]) do |thr|
      dist = { aai: {}, ani: {} }
      ref_ds.each_with_index do |ds, idx|
        if idx % cli[:threads] == thr
          cli.advance('Reading:', idx + 1, n, false) if thr == 0
          dist = read_bidirectional(ds, dist)
        end
      end
      if thr == 0
        cli.advance('Reading:', n, n, false)
        cli.say
      end
      File.open("#{tmp}/#{thr}.json", 'w') do |fh|
        fh.puts JSON.fast_generate(dist)
      end
    end

    # Merge pieces per thread
    dist = { aai: {}, ani: {} }
    cli[:threads].times do |i|
      cli.advance('Merging:', i + 1, cli[:threads], false)
      o = MiGA::Json.parse("#{tmp}/#{i}.json", symbolize: false)
      o.each { |k, v| dist[k.to_sym].merge!(v) }
    end
    cli.say
    FileUtils.rm_rf(tmp)

    # Write missing values (threaded)
    MiGA::Parallel.distribute(ref_ds, cli[:threads]) do |ds, idx, thr|
      cli.advance('Datasets:', idx + 1, n, false) if thr == 0
      save_bidirectional(ds, dist)
    end
    cli.say
  end

  ##
  # Perform distances operation with MiGA::Cli +cli+
  def check_dist(cli)
    p = cli.load_project<commit_msg>Safer doctor bidir thread dump<commit_after># @package MiGA
# @license Artistic-2.0

require 'miga/cli/action/doctor/base'

class MiGA::Cli::Action::Doctor < MiGA::Cli::Action
  include MiGA::Cli::Action::Doctor::Base

  def parse_cli
    cli.defaults = { threads: 1 }
    cli.defaults = Hash[@@OPERATIONS.keys.map { |i| [i, true] }]
    cli.parse do |opt|
      operation_n = Hash[@@OPERATIONS.map { |k, v| [v[0], k] }]
      cli.opt_object(opt, [:project])
      opt.on(
        '--ignore TASK1,TASK2', Array,
        'Do not perform the task(s) listed. Available tasks are:',
        * @@OPERATIONS.values.map { |v| "~ #{v[0]}: #{v[1]}" }
      ) { |v| v.map { |i| cli[operation_n[i]] = false } }
      opt.on(
        '--only TASK',
        'Perform only the specified task (see --ignore)'
      ) do |v|
        op_k = @@OPERATIONS.find { |_, i| i[0] == v.downcase }.first
        @@OPERATIONS.each_key { |i| cli[i] = false }
        cli[op_k] = true
      end
      opt.on(
        '-t', '--threads INT', Integer,
        "Concurrent threads to use. By default: #{cli[:threads]}"
      ) { |v| cli[:threads] = v }
    end
  end

  def perform
    p = cli.load_project
    @@OPERATIONS.keys.each do |k|
      send("check_#{k}", cli) if cli[k]
    end
  end

  @@OPERATIONS = {
    status: ['status', 'Update metadata status of all datasets'],
    db: ['databases', 'Check integrity of database files'],
    bidir: ['bidirectional', 'Check distances are bidirectional'],
    dist: ['distances', 'Check distance summary tables'],
    files: ['files', 'Check for outdated files'],
    cds: ['cds', 'Check for gzipped genes and proteins'],
    ess: ['essential-genes', 'Check for unarchived essential genes'],
    mts: ['mytaxa-scan', 'Check for unarchived MyTaxa scan'],
    start: ['start', 'Check for lingering .start files'],
    tax: ['taxonomy', 'Check for taxonomy consistency (not yet implemented)']
  }

  class << self
    ##
    # All supported operations
    def OPERATIONS
      @@OPERATIONS
    end
  end

  ##
  # Perform status operation with MiGA::Cli +cli+
  def check_status(cli)
    cli.say 'Updating metadata status'
    p = cli.load_project
    n = p.dataset_names.size
    (0 .. cli[:threads] - 1).map do |i|
      Process.fork do
        k = 0
        cli.load_project.each_dataset do |d|
          k += 1
          cli.advance('Datasets:', k, n, false) if i == 0
          d.recalculate_status if k % cli[:threads] == i
        end
      end
    end
    Process.waitall
    cli.say
  end

  ##
  # Perform databases operation with MiGA::Cli +cli+
  def check_db(cli)
    cli.say 'Checking integrity of databases'
    p = cli.load_project
    n = p.dataset_names.size
    (0 .. cli[:threads] - 1).map do |i|
      Process.fork do
        k = 0
        p.each_dataset do |d|
          k += 1
          cli.advance('Datasets:', k, n, false) if i == 0
          next unless k % cli[:threads] == i
          each_database_file(d) do |db_file, metric, result, _rank|
            check_sqlite3_database(db_file, metric) do
              cli.say(
                "  > Removing malformed database from #{d.name}:#{result}   "
              )
              File.unlink(db_file)
              r = d.result(result) or next
              [r.path(:done), r.path].each do |f|
                File.unlink(f) if File.exist?(f)
              end
            end
          end
        end
      end
    end
    Process.waitall
    cli.say
  end

  ##
  # Perform bidirectional operation with MiGA::Cli +cli+
  def check_bidir(cli)
    cli.say 'Checking if reference distances are bidirectional'
    project = cli.load_project
    ref_ds = project.each_dataset.select(&:ref?)
    ref_names = ref_ds.map(&:name)
    n = ref_ds.size

    # Read data first (threaded)
    tmp = File.join(project.path, 'doctor-bidirectional.tmp')
    FileUtils.mkdir_p(tmp)
    MiGA::Parallel.process(cli[:threads]) do |thr|
      dist = { aai: {}, ani: {} }
      ref_ds.each_with_index do |ds, idx|
        if idx % cli[:threads] == thr
          cli.advance('Reading:', idx + 1, n, false) if thr == 0
          dist = read_bidirectional(ds, dist)
        end
      end
      if thr == 0
        cli.advance('Reading:', n, n, false)
        cli.say
      end
      File.open("#{tmp}/#{thr}.json", 'w') do |fh|
        js = JSON.fast_generate(dist)
        fh.print js.slice!(0, 1024) until js.empty?
        fh.puts
        fh.flush # necessary for large threaded runs
      end
    end

    # Merge pieces per thread
    dist = { aai: {}, ani: {} }
    cli[:threads].times do |i|
      cli.advance('Merging:', i + 1, cli[:threads], false)
      o = MiGA::Json.parse("#{tmp}/#{i}.json", symbolize: false)
      o.each { |k, v| dist[k.to_sym].merge!(v) }
    end
    cli.say
    FileUtils.rm_rf(tmp)

    # Write missing values (threaded)
    MiGA::Parallel.distribute(ref_ds, cli[:threads]) do |ds, idx, thr|
      cli.advance('Datasets:', idx + 1, n, false) if thr == 0
      save_bidirectional(ds, dist)
    end
    cli.say
  end

  ##
  # Perform distances operation with MiGA::Cli +cli+
  def check_dist(cli)
    p = cli.load_project
{'fraction_non_alphanumeric': 0.14463667820069204, 'fraction_numerical': 0.005635195254572417, 'mean_word_length': 2.2663868259606073, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3195340', 'n_tokens_mistral': 3625, 'n_tokens_neox': 3443, 'n_words': 921}