<commit_before>{
  "output_dir_s3": "s3://<%= @attribute_dict[:bucket] %>/samples/<%= @project_id %>/<%= @sample_id %>/results",
  "targets": {
    "host_filter_out": [
      <% if @host_genome == "human" %>
        "gsnap_filter_1.fa"
        <% if @attribute_dict[:input_file_count] > 1 %>
          , "gsnap_filter_2.fa"
          , "gsnap_filter_merged.fa"
        <% end %>
     <% else %>
        "subsampled_1.fa"
        <% if @attribute_dict[:input_file_count] > 1 %>
          , "subsampled_2.fa"
          , "subsampled_merged.fa"
        <% end %>
     <% end %>
    ],
    "gsnap_out": [
      "gsnap.m8",
      "gsnap.deduped.m8",
      "gsnap.hitsummary.tab",
      "gsnap_counts.json"
    ],
    "refined_gsnap_out": [
      "gsnap.blast.m8",
      "gsnap.reassigned.m8",
      "gsnap_refined.hitsummary.tab",
      "refined_gsnap_counts.json",
      "gsnap_genus_accession_list.tab"
    ],
    "rapsearch2_out": [
      "rapsearch2.m8",
      "rapsearch2.deduped.m8",
      "rapsearch2.hitsummary.tab",
      "rapsearch2_counts.json"
    ],
    "refined_rapsearch2_out": [
      "rapsearch2.blast.m8",
      "rapsearch2.reassigned.m8",
      "rapsearch2_refined.hitsummary.tab",
      "refined_rapsearch2_counts.json",
      "rapsearch2_genus_accession_list.tab"
    ],
    "taxon_count_out": ["taxon_counts.json"],
    "annotated_out": ["annotated_merged.fa", "unidentified.fa"]
  },
  "steps": [
    {
      "in": ["host_filter_out"],
      "out": "gsnap_out",
      "class": "PipelineStepRunAlignmentRemotely",
      "module": "idseq_dag.steps.run_alignment_remotely",
      "additional_files": {
        "lineage_db": "<%= @attribute_dict[:lineage_db] %>",
        "accession2taxid_db": "<%= @attribute_dict[:accession2taxid_db] %>"
        <% if @attribute_dict[:skip_dedeuterostome_filter] == 0 %>
          ,"deuterostome_db": "<%= @attribute_dict[:deuterostome_db] %>"
        <% end %>
      },
      "additional_attributes": {
        "service": "gsnap",
        "chunks_in_flight": 32,
        "chunk_size": 15000,
        "max_concurrent": 2,
        "environment": "prod"
        <% if @attribute_dict[:index_dir_suffix] %>
          ,"index_dir_suffix": "<%= @attribute_dict[:index_dir_suffix] %>"
        <% end %>
      }
    },
    {
      "in": ["host_filter_out"],
      "out": "rapsearch2_out",
      "class": "PipelineStepRunAlignmentRemotely",
      "module": "idseq_dag.steps.run_alignment_remotely",
      "additional_files": {
        "lineage_db": "<%= @attribute_dict[:lineage_db] %>",
        "accession2taxid_db": "<%= @attribute_dict[:accession2taxid_db] %>"
        <% if @attribute_dict[:skip_dedeuterostome_filter] == 0 %>
          ,"deuterostome_db": "<%= @attribute_dict[:deuterostome_db] %>"
        <% end %>
      },
      "additional_attributes": {
        "service": "rapsearch2",
        "chunks_in_flight": 32,
        "chunk_size": 10000,
        "max_concurrent": 6,
        "environment": "prod"
        <% if @attribute_dict[:index_dir_suffix] %>
          ,"index_dir_suffix": "<%= @attribute_dict[:index_dir_suffix] %>"
        <% end %>
      }
    },
    {
      "in": ["gsnap_out"],
      "out": "refined_gsnap_out",
      "class": "PipelineStepReclassifyReads",
      "module": "idseq_dag.steps.reclassify_reads", 
      "additional_files": { 
        "lineage_db": "<%= @attribute_dict[:lineage_db] %>",
        "accession2taxid_db": "<%= @attribute_dict[:accession2taxid_db] %>",
        "loc_db": "<%= @attribute_dict[:nt_loc_db] %>"
      },
      "additional_attributes": {
        "db": "<%= @attribute_dict[:nt_db] %>",
        "db_type": "nt"
      }
    },
    {
      "in": ["rapsearch2_out"],
      "out": "refined_rapsearch2_out",
      "class": "PipelineStepReclassifyReads",
      "module": "idseq_dag.steps.reclassify_reads", 
      "additional_files": { 
        "lineage_db": "<%= @attribute_dict[:lineage_db] %>",<commit_msg>add host filter out to the reclassify step (#1486)<commit_after>{
  "output_dir_s3": "s3://<%= @attribute_dict[:bucket] %>/samples/<%= @project_id %>/<%= @sample_id %>/results",
  "targets": {
    "host_filter_out": [
      <% if @host_genome == "human" %>
        "gsnap_filter_1.fa"
        <% if @attribute_dict[:input_file_count] > 1 %>
          , "gsnap_filter_2.fa"
          , "gsnap_filter_merged.fa"
        <% end %>
     <% else %>
        "subsampled_1.fa"
        <% if @attribute_dict[:input_file_count] > 1 %>
          , "subsampled_2.fa"
          , "subsampled_merged.fa"
        <% end %>
     <% end %>
    ],
    "gsnap_out": [
      "gsnap.m8",
      "gsnap.deduped.m8",
      "gsnap.hitsummary.tab",
      "gsnap_counts.json"
    ],
    "refined_gsnap_out": [
      "gsnap.blast.m8",
      "gsnap.reassigned.m8",
      "gsnap_refined.hitsummary.tab",
      "gsnap_refined_counts.json",
      "gsnap_genus_accession_list.tab"
    ],
    "rapsearch2_out": [
      "rapsearch2.m8",
      "rapsearch2.deduped.m8",
      "rapsearch2.hitsummary.tab",
      "rapsearch2_counts.json"
    ],
    "refined_rapsearch2_out": [
      "rapsearch2.blast.m8",
      "rapsearch2.reassigned.m8",
      "rapsearch2_refined.hitsummary.tab",
      "rapsearch2_refined_counts.json",
      "rapsearch2_genus_accession_list.tab"
    ],
    "taxon_count_out": ["taxon_counts.json"],
    "annotated_out": ["annotated_merged.fa", "unidentified.fa"]
  },
  "steps": [
    {
      "in": ["host_filter_out"],
      "out": "gsnap_out",
      "class": "PipelineStepRunAlignmentRemotely",
      "module": "idseq_dag.steps.run_alignment_remotely",
      "additional_files": {
        "lineage_db": "<%= @attribute_dict[:lineage_db] %>",
        "accession2taxid_db": "<%= @attribute_dict[:accession2taxid_db] %>"
        <% if @attribute_dict[:skip_dedeuterostome_filter] == 0 %>
          ,"deuterostome_db": "<%= @attribute_dict[:deuterostome_db] %>"
        <% end %>
      },
      "additional_attributes": {
        "service": "gsnap",
        "chunks_in_flight": 32,
        "chunk_size": 15000,
        "max_concurrent": 2,
        "environment": "prod"
        <% if @attribute_dict[:index_dir_suffix] %>
          ,"index_dir_suffix": "<%= @attribute_dict[:index_dir_suffix] %>"
        <% end %>
      }
    },
    {
      "in": ["host_filter_out"],
      "out": "rapsearch2_out",
      "class": "PipelineStepRunAlignmentRemotely",
      "module": "idseq_dag.steps.run_alignment_remotely",
      "additional_files": {
        "lineage_db": "<%= @attribute_dict[:lineage_db] %>",
        "accession2taxid_db": "<%= @attribute_dict[:accession2taxid_db] %>"
        <% if @attribute_dict[:skip_dedeuterostome_filter] == 0 %>
          ,"deuterostome_db": "<%= @attribute_dict[:deuterostome_db] %>"
        <% end %>
      },
      "additional_attributes": {
        "service": "rapsearch2",
        "chunks_in_flight": 32,
        "chunk_size": 10000,
        "max_concurrent": 6,
        "environment": "prod"
        <% if @attribute_dict[:index_dir_suffix] %>
          ,"index_dir_suffix": "<%= @attribute_dict[:index_dir_suffix] %>"
        <% end %>
      }
    },
    {
      "in": ["gsnap_out", "host_filter_out"],
      "out": "refined_gsnap_out",
      "class": "PipelineStepReclassifyReads",
      "module": "idseq_dag.steps.reclassify_reads", 
      "additional_files": { 
        "lineage_db": "<%= @attribute_dict[:lineage_db] %>",
        "accession2taxid_db": "<%= @attribute_dict[:accession2taxid_db] %>",
        "loc_db": "<%= @attribute_dict[:nt_loc_db] %>"
      },
      "additional_attributes": {
        "db": "<%= @attribute_dict[:nt_db] %>",
        "db_type": "nt"
      }
    },
    {
      "in": ["rapsearch2_out", "host_filter_out"],
      "out": "refined_rapsearch2_out",
      "class": "PipelineStepReclassifyReads",
      "module": "idseq_dag.steps.reclassify_reads", 
      "additional_files": { 
        "lineage_db": "<%= @attribute_dict[:lineage_db] %>",
{'fraction_non_alphanumeric': 0.17731932666751044, 'fraction_numerical': 0.014428553347677509, 'mean_word_length': 2.7610661589719183, 'pattern_counts': {'":': 114, '<': 65, '<?xml version=': 0, '>': 69, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1200626', 'n_tokens_mistral': 3069, 'n_tokens_neox': 2836, 'n_words': 326}