<commit_before># filter viruses after ingest, criteria based on metadata
#  - viruses equal to or longer than 1701 bases
#  - viruses with exact dates
#  - viruses that are not egg-passaged
#  - a single sequence per virus strain, taken as first sequence in list

import os, re, time, datetime, csv, sys
from collections import defaultdict
from Bio import SeqIO
import numpy as np

class virus_filter(object):

	def __init__(self, alignment_file='', fasta_fields=None, date_spec='full', **kwargs):
		'''
		parameters:
		alignment_file   -- a FASTA sequence file with all viruses included
		fasta_fields     -- match between FASTA header index and virus dict field
		date_spec        -- if 'full', dates with day are required, if 'year', only year is accepted
		'''
		if fasta_fields is None:
			self.fasta_fields = {0:'strain', 1:'date' }
		else:
			self.fasta_fields = fasta_fields
		self.alignment_file = alignment_file
		self.viruses = self.parse_fasta(self.alignment_file)
		self.strain_lookup = {}
		self.outgroup = None
		self.date_spec = date_spec
		
	def parse_fasta(self, fasta):
		"""Parse FASTA file with default header formating"""
		viruses = []
		try:
			handle = open(fasta, 'r')
		except IOError:
			print fasta, "not found"
		else:
			for record in SeqIO.parse(handle, "fasta"):
				words = map(lambda x:x.strip(),record.description.replace(">","").split('|'))
				v = {key:words[ii] for ii, key in self.fasta_fields.iteritems()}
				v['seq']= str(record.seq)<commit_msg>added check to catch fasta field specification referring to non-existent fields in the fasta label<commit_after># filter viruses after ingest, criteria based on metadata
#  - viruses equal to or longer than 1701 bases
#  - viruses with exact dates
#  - viruses that are not egg-passaged
#  - a single sequence per virus strain, taken as first sequence in list

import os, re, time, datetime, csv, sys
from collections import defaultdict
from Bio import SeqIO
import numpy as np

class virus_filter(object):

	def __init__(self, alignment_file='', fasta_fields=None, date_spec='full', **kwargs):
		'''
		parameters:
		alignment_file   -- a FASTA sequence file with all viruses included
		fasta_fields     -- match between FASTA header index and virus dict field
		date_spec        -- if 'full', dates with day are required, if 'year', only year is accepted
		'''
		if fasta_fields is None:
			self.fasta_fields = {0:'strain', 1:'date' }
		else:
			self.fasta_fields = fasta_fields
		self.alignment_file = alignment_file
		self.viruses = self.parse_fasta(self.alignment_file)
		self.strain_lookup = {}
		self.outgroup = None
		self.date_spec = date_spec
		
	def parse_fasta(self, fasta):
		"""Parse FASTA file with default header formating"""
		viruses = []
		try:
			handle = open(fasta, 'r')
		except IOError:
			print fasta, "not found"
		else:
			for record in SeqIO.parse(handle, "fasta"):
				words = map(lambda x:x.strip(),record.description.replace(">","").split('|'))
				v = {key:words[ii] for ii, key in self.fasta_fields.iteritems() if ii<len(words)}
				v['seq']= str(record.seq)
{'fraction_non_alphanumeric': 0.10851202079272254, 'fraction_numerical': 0.003898635477582846, 'mean_word_length': 3.9983766233766236, 'pattern_counts': {'":': 0, '<': 4, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '740119', 'n_tokens_mistral': 1072, 'n_tokens_neox': 951, 'n_words': 329}