<commit_before>module Wrnap
  module Etl
    module Infernal
      class << self
        def parse_hit(output)
          Stockholm.fit_structure_to_sequence(
            *output.split(?\n).as do |infernal|
              [
                infernal.select { |line| line =~ /^.*\d+\s+.*\s+\d+\s*$/ }.last.match(/^.*\s+(.*)\s+\d+\s*$/)[1].upcase.gsub(/[^AUGC]/, ?.),
                convert_infernal_to_dot_bracket(infernal.find { |line| line =~ /CS\s*$/ }.gsub(/\s+CS\s*$/, "").strip)
              ]
            end
          )
        end

        def convert_infernal_to_dot_bracket(structure)
          # http://jalview-rnasupport.blogspot.com/2010/06/parsing-wuss-notation-of-rna-secondary.html
          structure.gsub(/[:,_-]/, ?.).gsub(/[<\[\{]/, ?().gsub(/[>\]\}]/, ?))
        end
      end
    end
  end
end<commit_msg>Parse an entire Infernal file.<commit_after>module Wrnap
  module Etl
    module Infernal
      class << self
        def parse_file(file)
          output = File.exist?(file) ? File.read(file) : file

          if output =~ /No hits detected that satisfy reporting thresholds/
            []
          else
            output.
              gsub(/^(.*\n)*Hit alignments:\n/, "").
              gsub(/Internal CM pipeline statistics summary:\n(.*\n)*$/, "").
              strip.split(?\n).reject(&:empty?).each_slice(10).map { |lines| parse_hit(lines.join(?\n)) }
          end
        end

        def parse_hit(output)
          name = if output =~ (identifier_regex = />>\s+(\S+)(.*\n){3}.*\s(\d+)\s+(\d+)\s+[\+-].*\n/)
            # This is a pretty fancy regex, and there's no guarantee that the data has this info, so let's just test the waters here.
            _, id, _, seq_from, seq_to, _ = output.match(identifier_regex).to_a
            "%s %d %d" % [id.split(?|).last, seq_from, seq_to]
          end

          Stockholm.fit_structure_to_sequence(
            *output.split(?\n).as do |infernal|
              [
                infernal.select { |line| line =~ /^.*\d+\s+.*\s+\d+\s*$/ }.last.match(/^.*\s+(\S+)\s+\d+\s*$/)[1].upcase.gsub(/[^AUGC]/, ?.),
                convert_infernal_to_dot_bracket(infernal.find { |line| line =~ /CS\s*$/ }.gsub(/\s+CS\s*$/, "").strip)
              ]
            end
          ).tap { |rna| rna.comment = name if name }
        end

        def convert_infernal_to_dot_bracket(structure)
          # http://jalview-rnasupport.blogspot.com/2010/06/parsing-wuss-notation-of-rna-secondary.html
          structure.gsub(/[:,_-]/, ?.).gsub(/[<\[\{]/, ?().gsub(/[>\]\}]/, ?))
        end
      end
    end
  end
end
{'fraction_non_alphanumeric': 0.1966010042487447, 'fraction_numerical': 0.00656624179219776, 'mean_word_length': 1.8091106290672452, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '891062', 'n_tokens_mistral': 938, 'n_tokens_neox': 893, 'n_words': 153}