<commit_before># Rakefile to create tables for execution manage

namespace :tables do
  # setup working dir
  workdir   = ENV['workdir'] || PROJ_ROOT
  table_dir = File.join(workdir, "tables")
  directory table_dir
  fastqc_dir     = ENV['fastqc_dir'] || File.join(workdir, "fastqc")
  sra_metadata   = ENV['sra_metadata_dir'] || File.join(table_dir, "sra_metadata")

  # path to list
  list_fastqc_finished = File.join(table_dir, "runs.done.tab")
  list_public_sra      = File.join(table_dir, "runs.public.tab")
  list_available       = File.join(table_dir, "experiments.available.tab")

  # set number of parallels
  Quanto::Records.set_number_of_parallels(NUM_OF_PARALLEL)
  Quanto::Records::IO.set_number_of_parallels(NUM_OF_PARALLEL)
  Quanto::Records::SRA.set_number_of_parallels(NUM_OF_PARALLEL)
  Quanto::Records::FastQC.set_number_of_parallels(NUM_OF_PARALLEL)

  # base task
  task :available => [
    sra_metadata,
    list_fastqc_finished,
    list_public_sra,
    list_available
  ]

  file sra_metadata do |t|
    Quanto::Records::SRA.download_sra_metadata(table_dir)
  end

  file list_fastqc_finished do |t|
    fastqc_records = Quanto::Records::FastQC.new(fastqc_dir)
    Quanto::Records::IO.write(fastqc_records.finished, t.name)
  end

  file list_public_sra => sra_metadata do |t|
    sra_records = Quanto::Records::SRA.new(sra_metadata)
    Quanto::Records::IO.write(sra_records.available, t.name)<commit_msg>getting metadata is task, not file task<commit_after># Rakefile to create tables for execution manage

namespace :tables do
  # setup working dir
  workdir      = ENV['workdir'] || PROJ_ROOT
  table_dir    = File.join(workdir, "tables")
  fastqc_dir   = ENV['fastqc_dir'] || File.join(workdir, "fastqc")
  sra_metadata = ENV['sra_metadata_dir'] || File.join(table_dir, "sra_metadata")

  # create directories if missing
  directory workdir
  directory table_dir
  directory fastqc_dir
  directory sra_metadata

  # path to list
  list_fastqc_finished = File.join(table_dir, "runs.done.tab")
  list_public_sra      = File.join(table_dir, "runs.public.tab")
  list_available       = File.join(table_dir, "experiments.available.tab")

  # set number of parallels
  Quanto::Records.set_number_of_parallels(NUM_OF_PARALLEL)
  Quanto::Records::IO.set_number_of_parallels(NUM_OF_PARALLEL)
  Quanto::Records::SRA.set_number_of_parallels(NUM_OF_PARALLEL)
  Quanto::Records::FastQC.set_number_of_parallels(NUM_OF_PARALLEL)

  # base task
  task :available => [
    :get_sra_metadata,
    list_fastqc_finished,
    list_public_sra,
    list_available
  ]

  task :get_sra_metadata => sra_matadata do |t|
    Quanto::Records::SRA.download_sra_metadata(table_dir)
  end

  file list_fastqc_finished do |t|
    fastqc_records = Quanto::Records::FastQC.new(fastqc_dir)
    Quanto::Records::IO.write(fastqc_records.finished, t.name)
  end

  file list_public_sra => sra_metadata do |t|
    sra_records = Quanto::Records::SRA.new(sra_metadata)
    Quanto::Records::IO.write(sra_records.available, t.name)
{'fraction_non_alphanumeric': 0.10871726881007623, 'fraction_numerical': 0.0, 'mean_word_length': 5.04809619238477, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 8, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1837733', 'n_tokens_mistral': 1150, 'n_tokens_neox': 1106, 'n_words': 180}