<commit_before>require 'mkfifo'
require 'tempfile'

module Bio
  module Gadget
    class Fq1l < Thor

      ## Change to 't3 TRIMMED SEQ1 SEQ2 ...', which internally supports 'grep -e SEQ1 -e SEQ2 ...'
      desc 't3 SEQ TRIMMED', 'Trim sequences that match the 3\'-end with SEQ'
      
      def t3(seq, trimmed)
        fifo = Dir::Tmpname.create(['rbg.fq1l.t3.', '.fq1l']) {  }
        File.mkfifo(fifo)
        prefix = options.prefix_coreutils 
        pid = Kernel.fork do
          exec "#{prefix}tee #{fifo} | #{prefix}grep -v '#{seq}\t+'"
        end
        range = 0..(-seq.length-1)
        out = open(trimmed, 'w')
        open("| #{prefix}grep '#{seq}\t+' #{fifo}").each do |line|
          acc, raw, tmp, qual = line.rstrip.split /\t/
          out.puts [acc, raw[range], tmp, qual[range]].join("\t")
        end
        out.close
        Process.wait(pid)
      ensure
        File.unlink(fifo)
      end
      
    end
  end
end<commit_msg>support multiple sequences by "fq1l t3"<commit_after>require 'mkfifo'
require 'tempfile'

module Bio
  module Gadget
    class Fq1l < Thor

      desc 't3 TRIMMED SEQ1 [SEQ2 ...]', 'Trim sequences that match the 3\'-end with SEQs'
      
      def t3(trimmed, *seqs)
        len = seqs[0].length
        seqs.each do |seq|
          abort "SEQs must be a same length." if seq.length != len
        end
        patterns = (seqs.map { |seq| "-e '#{seq}\t'" }).join ' '
        #
        fifo = Dir::Tmpname.create(['rbg.fq1l.t3.', '.fq1l']) {  }
        File.mkfifo(fifo)
        prefix = options.prefix_coreutils 
        pid = Kernel.fork do
          exec "#{prefix}tee #{fifo} | #{prefix}grep -v #{patterns}"
        end
        #
        range = 0..(-len-1)
        out = open(trimmed, 'w')
        open("| #{prefix}grep #{patterns} #{fifo}").each do |line|
          acc, raw, tmp, qual = line.rstrip.split /\t/
          out.puts [acc, raw[range], tmp, qual[range]].join("\t")
        end
        out.close
        Process.wait(pid)
      ensure
        File.unlink(fifo)
      end
      
    end
  end
end
{'fraction_non_alphanumeric': 0.1513539651837524, 'fraction_numerical': 0.013539651837524178, 'mean_word_length': 1.877607788595271, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6168588', 'n_tokens_mistral': 752, 'n_tokens_neox': 736, 'n_words': 187}