<commit_before># simple script to run required operations to
# 1. Upload GISAID XLS and FASTA to database
# 2. Update genetic groupings
# 3. Download FASTAs from database
# 4. Copy FASTAs to nextflu directory
# 5. Download titer tables from database
# 6. Copy titer tables to nextflu directory
# Run from base nextstrain-db directory with python flu/pipeline.py
# Requires files gisaid_epiflu.xls and gisaid_epiflu.fasta in data/

import os

# Upload GISAID XLS and FASTA to database
os.system("python vdb/flu_upload.py -db vdb -v flu --source gisaid --fname gisaid_epiflu")

# Update genetic groupings
os.system("python vdb/flu_update.py -db vdb -v flu --update_groupings")

# Download FASTAs from database
os.system("python vdb/flu_download.py -db vdb -v flu --select locus:HA lineage:seasonal_h3n2 --fstem h3n2")
os.system("python vdb/flu_download.py -db vdb -v flu --select locus:HA lineage:seasonal_h1n1pdm --fstem h1n1pdm")
os.system("python vdb/flu_download.py -db vdb -v flu --select locus:HA lineage:seasonal_vic --fstem vic")
os.system("python vdb/flu_download.py -db vdb -v flu --select locus:HA lineage:seasonal_yam --fstem yam")

# Copy FASTAs to nextflu directory
os.system("cp data/h3n2.fasta ../nextflu/augur/data/")
os.system("cp data/h1n1pdm.fasta ../nextflu/augur/data/")
os.system("cp data/vic.fasta ../nextflu/augur/data/")
os.system("cp data/yam.fasta ../nextflu/augur/data/")

# Download titer tables from database
os.system("python tdb/download.py -db tdb -v flu --ftype augur --subtype h3n2")
os.system("python tdb/download.py -db tdb -v flu --ftype augur --subtype h1n1pdm")
os.system("python tdb/download.py -db tdb -v flu --ftype augur --subtype vic")
os.system("python tdb/download.py -db tdb -v flu --ftype augur --subtype yam")

# Copy titer tables to nextflu directory
os.system("cp data/h3n2_hi_strains.txt ../nextflu/augur/data/")
os.system("cp data/h3n2_hi_titers.txt ../nextflu/augur/data/")
os.system("cp data/h1n1pdm_hi_strains.txt ../nextflu/augur/data/")
os.system("cp data/h1n1pdm_hi_titers.txt ../nextflu/augur/data/")
os.system("cp data/vic_hi_strains.txt ../nextflu/augur/data/")
os.system("cp data/vic_hi_titers.txt ../nextflu/augur/data/")
os.system("cp data/yam_hi_strains.txt ../nextflu/augur/data/")
os.system("cp data/yam_hi_titers.txt ../nextflu/augur/data/")<commit_msg>Fix flu pipeline.<commit_after># simple script to run required operations to
# 1. Upload GISAID XLS and FASTA to database
# 2. Update genetic groupings
# 3. Download FASTAs from database
# 4. Copy FASTAs to nextflu directory
# 5. Download titer tables from database
# 6. Copy titer tables to nextflu directory
# Run from base nextstrain-db directory with python flu/pipeline.py
# Requires files gisaid_epiflu.xls and gisaid_epiflu.fasta in data/

import os

# Upload GISAID XLS and FASTA to database
os.system("python vdb/flu_upload.py -db vdb -v flu --source gisaid --fname gisaid_epiflu")

# Update genetic groupings
os.system("python vdb/flu_update.py -db vdb -v flu --update_groupings")

# Download FASTAs from database
os.system("python vdb/flu_download.py -db vdb -v flu --select locus:HA lineage:seasonal_h3n2 --fstem h3n2")
os.system("python vdb/flu_download.py -db vdb -v flu --select locus:HA lineage:seasonal_h1n1pdm --fstem h1n1pdm")
os.system("python vdb/flu_download.py -db vdb -v flu --select locus:HA lineage:seasonal_vic --fstem vic")
os.system("python vdb/flu_download.py -db vdb -v flu --select locus:HA lineage:seasonal_yam --fstem yam")

# Copy FASTAs to nextflu directory
os.system("cp data/h3n2.fasta ../nextflu/augur/data/")
os.system("cp data/h1n1pdm.fasta ../nextflu/augur/data/")
os.system("cp data/vic.fasta ../nextflu/augur/data/")
os.system("cp data/yam.fasta ../nextflu/augur/data/")

# Download titer tables from database
os.system("python tdb/download.py -db tdb -v flu --ftype augur --subtype h3n2")
os.system("python tdb/download.py -db tdb -v flu --ftype augur --subtype h1n1pdm")
os.system("python tdb/download.py -db tdb -v flu --ftype augur --subtype vic")
os.system("python tdb/download.py -db tdb -v flu --ftype augur --subtype yam")

# Copy titer tables to nextflu directory
os.system("cp data/h3n2_hi_strains.tsv ../nextflu/augur/data/")
os.system("cp data/h3n2_hi_titers.tsv ../nextflu/augur/data/")
os.system("cp data/h1n1pdm_hi_strains.tsv ../nextflu/augur/data/")
os.system("cp data/h1n1pdm_hi_titers.tsv ../nextflu/augur/data/")
os.system("cp data/vic_hi_strains.tsv ../nextflu/augur/data/")
os.system("cp data/vic_hi_titers.tsv ../nextflu/augur/data/")
os.system("cp data/yam_hi_strains.tsv ../nextflu/augur/data/")
os.system("cp data/yam_hi_titers.tsv ../nextflu/augur/data/")
{'fraction_non_alphanumeric': 0.13876936316695354, 'fraction_numerical': 0.012908777969018933, 'mean_word_length': 4.07532751091703, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 24, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2172583', 'n_tokens_mistral': 1837, 'n_tokens_neox': 1754, 'n_words': 457}