<commit_before>require 'find'
require 'forwardable'

module SequenceServer
  # Smart `makeblastdb` wrapper: recursively scans database directory determining
  # which files need to be formatted or re-formatted.
  #
  # Example usage:
  #
  #   makeblastdb = MAKEBLASTDB.new(database_dir)
  #   makeblastdb.scan && makeblastdb.run
  #
  class MAKEBLASTDB
    # Expected extensions for nucleotide and proteind databases created with or
    # without -parse_seqids and -hash_index options.
    EXPECTED_EXTENSIONS = {
      'nucleotide' => {
        %w[nal]                                         => %w[alias],
        %w[nhr nin nsq]                                 => %w[v4],
        %w[nhr nin nog nsd nsi nsq]                     => %w[v4 parse_seqids],
        %w[nhd nhi nhr nin nog nsd nsi nsq]             => %w[v4 parse_seqids hash_index],
        %w[ndb nhr nin not nsq ntf nto]                 => %w[v5],
        %w[ndb nhr nin nog nos not nsq ntf nto]         => %w[v5 parse_seqids],
        %w[ndb nhd nhi nhr nin nog nos not nsq ntf nto] => %w[v5 parse_seqids hash_index],
        %w[ndb nhd nhi nhr nin nog not nsq ntf nto]     => %w[v5 hash_index]
      },
      'protein' => {
        %w[pal]                                         => %w[alias],
        %w[phr pin psq]                                 => %w[v4],
        %w[phr pin pog psd psi psq]                     => %w[v4 parse_seqids],
        %w[phd phi phr pin pog psd psi psq]             => %w[v4 parse_seqids hash_index],
        %w[pdb phr pin pot psq ptf pto]                 => %w[v5],
        %w[pdb phr pin pog pos pot psq ptf pto]         => %w[v5 parse_seqids],
        %w[pdb phd phi phr pin pog pos pot psq ptf pto] => %w[v5 parse_seqids hash_index],
        %w[pdb phd phi phr pin pog pot psq ptf pto]     => %w[v5 hash_index]
      }
    }

    extend Forwardable

    def_delegators SequenceServer, :config, :sys

    def initialize(database_dir)
      @database_dir = database_dir
    end

    attr_reader :database_dir
    attr_reader :formatted_fastas
    attr_reader :fastas_to_format
    attr_reader :fastas_to_reformat

    # Scans the database directory to determine which FASTA files require
    # formatting or re-formatting.
    #
    # Returns `true` if there are files to (re-)format, `false` otherwise.
    def scan
      # We need to know the list of formatted FASTAs as reported by blastdbcmd
      # first. This is required to determine both unformatted FASTAs and those
      # that require reformatting.
      @formatted_fastas = []
      determine_formatted_fastas

      # Now determine FASTA files that are unformatted or require reformatting.
      @fastas_to_format = []
      determine_unformatted_fastas
      @fastas_to_reformat = []
      determine_fastas_to_reformat

      # Return true if there are files to be (re-)formatted or false otherwise.
      !@fastas_to_format.empty? || !@fastas_to_reformat.empty?
    end

    # Returns true if at least one database in database directory is formatted.
    def any_formatted?
      !@formatted_fastas.empty?
    end

    # Returns true if there is at least one unformatted FASTA in the databases
    # directory.
    def any_unformatted?
      !@fastas_to_format.empty?
    end

    # Returns true if the databases directory contains one or more incompatible
    # databases.
    #
    # Note that it is okay to only use V4 databases or only V5 databases.
    # Incompatibility arises when they are mixed.
    def any_incompatible?
      return false if @formatted_fastas.all? { |ff| ff.v4? || ff.alias? }
      return false if @formatted_fastas.all? { |ff| ff.v5? || ff.alias? }
      true
    end

    # Runs makeblastdb on each file in `@fastas_to_format` and
    # `@fastas_to_reformat`. Will do nothing unless `#scan`
    # has been run before.
    def run
      format
      reformat
    end

    # Format any unformatted FASTA files in database directory. Returns Array
    # of files that were formatted.
    def format
      # Make the intent clear as well as ensure the program won't crash if we
      # accidentally call format before calling scan.
      return unless @fastas_to_format
      @fastas_to_format.select do |path, title, type|
        make_blast_database('format', path, title, type)
      end
    end

    # Re-format databases that require reformatting. Returns Array of files
    # that were reformatted.
    def reformat
      # Make the intent clear as well as ensure the program won't crash if
      # we accidentally call reformat before calling scan.
      return unless @fastas_to_reformat
      @fastas_to_reformat.select do |path, title, type, non_parse_seqids|
        make_blast_database('reformat', path, title, type, non_parse_seqids)
      end
    end

    private

    # Determines which FASTA files in the database directory are already
    # formatted. Adds to @formatted_fastas.
    def determine_formatted_fastas
      blastdbcmd.each_line do |line|
        path, title, type, *rest = line.split("\t")
        type.downcase!
        next if multipart_database_name?(path)
        rest << get_format(path, type)
        rest << get_categories(path)
        @formatted_fastas << Database.new(path, title, type, *rest)
      end
    end

    def get_format(path, type)
      exts = Dir["#{path}.*"].map { |p| p.split('.').last }.sort
      EXPECTED_EXTENSIONS[type][exts]
    end

    # Determines which FASTA files in the database directory require
    # reformatting. Adds to @fastas_to_format.
    def determine_fastas_to_reformat
      @formatted_fastas.each do |ff|
        if ff.v4? || ff.non_parse_seqids?
          @fastas_to_reformat << [ff.path, ff.title, ff.type, ff.non_parse_seqids?]
        end
      end
    end

    # Determines which FASTA files in the database directory are
    # unformatted. Adds to @fastas_to_format.
    def determine_unformatted_fastas
      Find.find(database_dir) do |path|<commit_msg>Default return value of MAKEBLASTDB#get_format should be [], not nil<commit_after>require 'find'
require 'forwardable'

module SequenceServer
  # Smart `makeblastdb` wrapper: recursively scans database directory determining
  # which files need to be formatted or re-formatted.
  #
  # Example usage:
  #
  #   makeblastdb = MAKEBLASTDB.new(database_dir)
  #   makeblastdb.scan && makeblastdb.run
  #
  class MAKEBLASTDB
    # Expected extensions for nucleotide and proteind databases created with or
    # without -parse_seqids and -hash_index options.
    EXPECTED_EXTENSIONS = {
      'nucleotide' => {
        %w[nal]                                         => %w[alias],
        %w[nhr nin nsq]                                 => %w[v4],
        %w[nhr nin nog nsd nsi nsq]                     => %w[v4 parse_seqids],
        %w[nhd nhi nhr nin nog nsd nsi nsq]             => %w[v4 parse_seqids hash_index],
        %w[ndb nhr nin not nsq ntf nto]                 => %w[v5],
        %w[ndb nhr nin nog nos not nsq ntf nto]         => %w[v5 parse_seqids],
        %w[ndb nhd nhi nhr nin nog nos not nsq ntf nto] => %w[v5 parse_seqids hash_index],
        %w[ndb nhd nhi nhr nin nog not nsq ntf nto]     => %w[v5 hash_index]
      },
      'protein' => {
        %w[pal]                                         => %w[alias],
        %w[phr pin psq]                                 => %w[v4],
        %w[phr pin pog psd psi psq]                     => %w[v4 parse_seqids],
        %w[phd phi phr pin pog psd psi psq]             => %w[v4 parse_seqids hash_index],
        %w[pdb phr pin pot psq ptf pto]                 => %w[v5],
        %w[pdb phr pin pog pos pot psq ptf pto]         => %w[v5 parse_seqids],
        %w[pdb phd phi phr pin pog pos pot psq ptf pto] => %w[v5 parse_seqids hash_index],
        %w[pdb phd phi phr pin pog pot psq ptf pto]     => %w[v5 hash_index]
      }
    }

    extend Forwardable

    def_delegators SequenceServer, :config, :sys

    def initialize(database_dir)
      @database_dir = database_dir
    end

    attr_reader :database_dir
    attr_reader :formatted_fastas
    attr_reader :fastas_to_format
    attr_reader :fastas_to_reformat

    # Scans the database directory to determine which FASTA files require
    # formatting or re-formatting.
    #
    # Returns `true` if there are files to (re-)format, `false` otherwise.
    def scan
      # We need to know the list of formatted FASTAs as reported by blastdbcmd
      # first. This is required to determine both unformatted FASTAs and those
      # that require reformatting.
      @formatted_fastas = []
      determine_formatted_fastas

      # Now determine FASTA files that are unformatted or require reformatting.
      @fastas_to_format = []
      determine_unformatted_fastas
      @fastas_to_reformat = []
      determine_fastas_to_reformat

      # Return true if there are files to be (re-)formatted or false otherwise.
      !@fastas_to_format.empty? || !@fastas_to_reformat.empty?
    end

    # Returns true if at least one database in database directory is formatted.
    def any_formatted?
      !@formatted_fastas.empty?
    end

    # Returns true if there is at least one unformatted FASTA in the databases
    # directory.
    def any_unformatted?
      !@fastas_to_format.empty?
    end

    # Returns true if the databases directory contains one or more incompatible
    # databases.
    #
    # Note that it is okay to only use V4 databases or only V5 databases.
    # Incompatibility arises when they are mixed.
    def any_incompatible?
      return false if @formatted_fastas.all? { |ff| ff.v4? || ff.alias? }
      return false if @formatted_fastas.all? { |ff| ff.v5? || ff.alias? }
      true
    end

    # Runs makeblastdb on each file in `@fastas_to_format` and
    # `@fastas_to_reformat`. Will do nothing unless `#scan`
    # has been run before.
    def run
      format
      reformat
    end

    # Format any unformatted FASTA files in database directory. Returns Array
    # of files that were formatted.
    def format
      # Make the intent clear as well as ensure the program won't crash if we
      # accidentally call format before calling scan.
      return unless @fastas_to_format
      @fastas_to_format.select do |path, title, type|
        make_blast_database('format', path, title, type)
      end
    end

    # Re-format databases that require reformatting. Returns Array of files
    # that were reformatted.
    def reformat
      # Make the intent clear as well as ensure the program won't crash if
      # we accidentally call reformat before calling scan.
      return unless @fastas_to_reformat
      @fastas_to_reformat.select do |path, title, type, non_parse_seqids|
        make_blast_database('reformat', path, title, type, non_parse_seqids)
      end
    end

    private

    # Determines which FASTA files in the database directory are already
    # formatted. Adds to @formatted_fastas.
    def determine_formatted_fastas
      blastdbcmd.each_line do |line|
        path, title, type, *rest = line.split("\t")
        type.downcase!
        next if multipart_database_name?(path)
        rest << get_format(path, type)
        rest << get_categories(path)
        @formatted_fastas << Database.new(path, title, type, *rest)
      end
    end

    def get_format(path, type)
      exts = Dir["#{path}.*"].map { |p| p.split('.').last }.sort
      EXPECTED_EXTENSIONS[type][exts] || []
    end

    # Determines which FASTA files in the database directory require
    # reformatting. Adds to @fastas_to_format.
    def determine_fastas_to_reformat
      @formatted_fastas.each do |ff|
        if ff.v4? || ff.non_parse_seqids?
          @fastas_to_reformat << [ff.path, ff.title, ff.type, ff.non_parse_seqids?]
        end
      end
    end

    # Determines which FASTA files in the database directory are
    # unformatted. Adds to @fastas_to_format.
    def determine_unformatted_fastas
      Find.find(database_dir) do |path|
{'fraction_non_alphanumeric': 0.08029502975442125, 'fraction_numerical': 0.0031849803034112814, 'mean_word_length': 2.3763440860215055, 'pattern_counts': {'":': 0, '<': 19, '<?xml version=': 0, '>': 39, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 25, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2626885', 'n_tokens_mistral': 3842, 'n_tokens_neox': 3723, 'n_words': 1199}