<commit_before># -*- coding: utf-8 -*-

require 'rake'
require 'parallel'
require 'zip'
require 'ciika'

module Quanto
  class Records
    class << self
      def num_of_parallels
        NUM_OF_PARALLELS || 4
      end

      def target_archives
        # ["DRR","ERR","SRR"] # complete list
        ["DRR"]
      end
    end

    def initialize(sra_available, fastqc_finished)
      @sra_available = sra_available
      @fastqc_finished = fastqc_finished
      @nop = Quanto::Records.num_of_parallels
    end

    def available
      finished = runids_finished
      available_record = Parallel.map(@sra_available, :in_threads => @nop) do |record|
        run_id = record[0]
        experiment_record = [record[2], record[1], record[3], record[4]]
        experiment_record if !finished.include?(run_id)
      end
      available_record.uniq.compact
    end

    def runids_finished
      runids = Parallel.map(@fastqc_finished, :in_threads => @nop) do |record|
        fastqc_path = record.split("\t")[0]
        fastqc_path.split("/").last.split("_")[0]
      end
      runids.uniq
    end
  end
end<commit_msg>implemented published before and after, need to be diy<commit_after># -*- coding: utf-8 -*-

require 'rake'
require 'parallel'
require 'zip'
require 'ciika'

module Quanto
  class Records
    class << self
      def num_of_parallels
        NUM_OF_PARALLELS || 4
      end

      def target_archives
        # ["DRR","ERR","SRR"] # complete list
        ["DRR"]
      end
    end

    def initialize(sra_available, fastqc_finished)
      @sra_available = sra_available
      @fastqc_finished = fastqc_finished
      @nop = Quanto::Records.num_of_parallels
    end

    def published_before(date)
      finished = runids_finished
      available_record = Parallel.map(@sra_available, :in_threads => @nop) do |record|
        run_id = record[0]
        is_older = DateTime.parse(record[3]) < DateTime.parse(date)
        experiment_record if !finished.include?(run_id) && is_older
      end
      available_record.compact.uniq
    end

    def published_after(date)
      finished = runids_finished
      available_record = Parallel.map(@sra_available, :in_threads => @nop) do |record|
        run_id = record[0]
        is_newer = DateTime.parse(record[3]) > DateTime.parse(date)
        experiment_record if !finished.include?(run_id) && is_newer
      end
      available_record.compact.uniq
    end

    def available
      finished = runids_finished
      available_record = Parallel.map(@sra_available, :in_threads => @nop) do |record|
        run_id = record[0]
        experiment_record if !finished.include?(run_id)
      end
      available_record.compact.uniq
    end

    def experiment_record(record)
      [record[2], record[1], record[3], record[4]]
    end

    def runids_finished
      runids = Parallel.map(@fastqc_finished, :in_threads => @nop) do |record|
        fastqc_path = record.split("\t")[0]
        fastqc_path.split("/").last.split("_")[0]
      end
      runids.uniq
    end
  end
end
{'fraction_non_alphanumeric': 0.10762627930009905, 'fraction_numerical': 0.007263123142951469, 'mean_word_length': 2.7546468401486988, 'pattern_counts': {'":': 0, '<': 8, '<?xml version=': 0, '>': 10, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1318576', 'n_tokens_mistral': 1005, 'n_tokens_neox': 974, 'n_words': 211}