<commit_before>import time, argparse,re,os
from virus_filter import flu_filter
from virus_clean import virus_clean
from tree_refine import tree_refine
from tree_mutations import tree_mutations
from process import process, virus_config
from Bio import SeqIO
from Bio.Seq import Seq
from Bio.Align import MultipleSeqAlignment
import numpy as np
from itertools import izip


# numbering starting at HA1 start, adding sp to obtain numbering from methionine
sp = 16
epitope_mask = np.fromstring(sp*"0"+"0000000000000000000000000000000000000000000011111011011001010011000100000001001011110011100110101000001100000100000001000110101011111101011010111110001010011111000101011011111111010010001111101110111001010001110011111111000000111110000000101010101110000000000011100100000001011011100000000000001001011000110111111000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000", dtype='S1')
receptor_binding_sites = map(lambda x:x+sp-1, [145, 155, 156, 158, 159, 189, 193])


virus_config.update({
	# data source and sequence parsing/cleaning/processing
	'virus':'H3N2',
	'alignment_file':'data/H3N2_gisaid_epiflu_sequence.fasta',
	'outgroup':'A/Beijing/32/1992',
	#'force_include':'source-data/HI_strains.txt',
	'force_include_all':False,
	'max_global':True,   # sample as evenly as possible from different geographic regions 
	'cds':[0,None], # define the HA1 start i n 0 numbering
	'n_iqd':6,
	# define relevant clades in canonical HA1 numbering (+1)
	# numbering starting at HA1 start, adding sp to obtain numbering from methionine
	'clade_designations': { "3c3.a":[(128+sp,'A'), (142+sp,'G'), (159+sp,'S')],
						   "3c3":   [(128+sp,'A'), (142+sp,'G'), (159+sp,'F')],
						   "3c2.a": [(144+sp,'S'), (159+sp,'Y'), (225+sp,'D'), (311+sp,'H'), (489+sp,'N')],
						   "3c2":   [(144+sp,'N'), (159+sp,'F'), (225+sp,'N'), (489+sp,'N'), (142+sp, 'R')],
						   "3c3.b":   [(83+sp,'R'), (261+sp,'Q'), (62+sp,'K'), (122+sp,'D')]
							},
	'html_vars': {'coloring': 'ep, ne, rb, lbi, dfreq, region, date',
				   'gtplaceholder': 'HA1 positions...',
					'freqdefault': '3c2.a, 3c3.a'},
	'js_vars': {'LBItau': 0.0005, 'LBItime_window': 0.5, 'dfreq_dn':2},
	})


class H3N2_filter(flu_filter):
	def __init__(self,min_length = 987, **kwargs):
		'''<commit_msg>Include FASTA fields in H3N2_process.py.<commit_after>import time, argparse,re,os
from virus_filter import flu_filter
from virus_clean import virus_clean
from tree_refine import tree_refine
from tree_mutations import tree_mutations
from process import process, virus_config
from Bio import SeqIO
from Bio.Seq import Seq
from Bio.Align import MultipleSeqAlignment
import numpy as np
from itertools import izip


# numbering starting at HA1 start, adding sp to obtain numbering from methionine
sp = 16
epitope_mask = np.fromstring(sp*"0"+"0000000000000000000000000000000000000000000011111011011001010011000100000001001011110011100110101000001100000100000001000110101011111101011010111110001010011111000101011011111111010010001111101110111001010001110011111111000000111110000000101010101110000000000011100100000001011011100000000000001001011000110111111000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000", dtype='S1')
receptor_binding_sites = map(lambda x:x+sp-1, [145, 155, 156, 158, 159, 189, 193])


virus_config.update({
	# data source and sequence parsing/cleaning/processing
	'virus':'H3N2',
	'alignment_file':'data/H3N2_gisaid_epiflu_sequence.fasta',
	'fasta_fields':{0:'strain', 1:'isolate_id', 3:'passage', 5:'date', 7:'lab', 8:"accession"},
	'outgroup':'A/Beijing/32/1992',
	#'force_include':'source-data/HI_strains.txt',
	'force_include_all':False,
	'max_global':True,   # sample as evenly as possible from different geographic regions 
	'cds':[0,None], # define the HA1 start i n 0 numbering
	'n_iqd':6,
	# define relevant clades in canonical HA1 numbering (+1)
	# numbering starting at HA1 start, adding sp to obtain numbering from methionine
	'clade_designations': { "3c3.a":[(128+sp,'A'), (142+sp,'G'), (159+sp,'S')],
						   "3c3":   [(128+sp,'A'), (142+sp,'G'), (159+sp,'F')],
						   "3c2.a": [(144+sp,'S'), (159+sp,'Y'), (225+sp,'D'), (311+sp,'H'), (489+sp,'N')],
						   "3c2":   [(144+sp,'N'), (159+sp,'F'), (225+sp,'N'), (489+sp,'N'), (142+sp, 'R')],
						   "3c3.b":   [(83+sp,'R'), (261+sp,'Q'), (62+sp,'K'), (122+sp,'D')]
							},
	'html_vars': {'coloring': 'ep, ne, rb, lbi, dfreq, region, date',
				   'gtplaceholder': 'HA1 positions...',
					'freqdefault': '3c2.a, 3c3.a'},
	'js_vars': {'LBItau': 0.0005, 'LBItime_window': 0.5, 'dfreq_dn':2},
	})


class H3N2_filter(flu_filter):
	def __init__(self,min_length = 987, **kwargs):
		'''
{'fraction_non_alphanumeric': 0.146669341894061, 'fraction_numerical': 0.27447833065810595, 'mean_word_length': 7.092532467532467, 'pattern_counts': {'":': 10, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 20, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1846187', 'n_tokens_mistral': 2844, 'n_tokens_neox': 1632, 'n_words': 360}