<commit_before>##
# uniquify - uniquify a fasta file, also output table with sequence_id -> number of reads
#

module Lederhosen
  class CLI
    desc 'uniquify',
      'uniquify a fasta file and generate a table with sequence_id -> abundance'

    method_option :input,     :type => :string, :required => true
    method_option :output,    :type => :string, :required => true
    method_option :table_out, :type => :string, :required => true

    def uniquify
      input     = options[:input]
      output    = options[:output]
      table_out = options[:table_out]

      ohai "uniquifying #{input} to #{output} w/ table #{table_out}"

      sequence_counts = Hash.new { |h, k| h[k] = 0 }

      out = File.open(output, 'w')

      no_records = `grep -c '^>' #{input}`.split.first.to_i
      pbar = ProgressBar.new 'loading', no_records
      File.open(input) do |handle|
        Dna.new(handle).each do |record|
          pbar.inc
          unless sequence_counts.has_key? record.sequence
            out.puts record
          end
          sequence_counts[record.sequence] += 1
        end
      end

      pbar.finish
      out.close

      # write table
      pbar = ProgressBar.new 'table', no_records
      File.open(table_out, 'w') do |out|
        sequence_counts.each_pair do |sequence, count|
          pbar.inc
          out.puts "#{sequence.name},#{count}"
        end
      end
      pbar.finish
      kept = sequence_counts.keys.size
      total = sequence_counts.values.inject(:+)
      ohai "kept #{kept} out of #{total} reads (#{100*kept/total.to_f})"
    end
  end
end<commit_msg>uniquify prints table with ids<commit_after>##
# uniquify - uniquify a fasta file, also output table with sequence_id -> number of reads
#

module Lederhosen
  class CLI
    desc 'uniquify',
      'uniquify a fasta file and generate a table with sequence_id -> abundance'

    method_option :input,     :type => :string, :required => true
    method_option :output,    :type => :string, :required => true
    method_option :table_out, :type => :string, :required => true

    def uniquify
      input     = options[:input]
      output    = options[:output]
      table_out = options[:table_out]

      ohai "uniquifying #{input} to #{output} w/ table #{table_out}"

      sequence_counts = Hash.new { |h, k| h[k] = 0 }
      sequence_to_id = Hash.new

      out = File.open(output, 'w')

      no_records = `grep -c '^>' #{input}`.split.first.to_i
      pbar = ProgressBar.new 'loading', no_records

      File.open(input) do |handle|
        Dna.new(handle).each do |record|
          pbar.inc
          unless sequence_counts.has_key? record.sequence
            # store the sequence and id so we can have ids in the
            # table. If the file is sorted by length then this
            # should also be a seed sequence.
            sequence_to_id[record.sequence] = record.name
            out.puts record
          end
          sequence_counts[record.sequence] += 1
        end
      end

      pbar.finish
      out.close

      # write table
      pbar = ProgressBar.new 'table', no_records
      File.open(table_out, 'w') do |out|
        sequence_counts.each_pair do |sequence, count|
          pbar.inc
          id = sequence_to_id[sequence]
          out.puts "#{id},#{count}"
        end
      end
      pbar.finish
      kept = sequence_counts.keys.size
      total = sequence_counts.values.inject(:+)
      ohai "kept #{kept} out of #{total} reads (#{100*kept/total.to_f})"
    end
  end
end
{'fraction_non_alphanumeric': 0.11104783599088838, 'fraction_numerical': 0.0028473804100227792, 'mean_word_length': 2.289325842696629, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 21, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5878169', 'n_tokens_mistral': 1120, 'n_tokens_neox': 1094, 'n_words': 309}