<commit_before>#!/usr/bin/python

## adapted from <NAME>'s code. it can also be used with alignment sequences with models, e.g., rdp alignment files. Wild card "*" could be used with for multiple input sequence files. 

from Bio import SeqIO
import numpy
import sys
import re
import gzip

def main(mode, fasta_files):
	num_seqs = 0
	lengths = []

	for fasta_file in fasta_files:
		for seq in SeqIO.parse(gzip.open(fasta_file), "fastq"):
			if seq.id[0] == "#":
				continue

			length = 0
			seq_string = str(seq.seq)
			if mode == "model":
				length = len(re.sub(non_model_regex, "", seq_string))<commit_msg>modified seq_stats_fastq_gz for fastq files<commit_after>#!/usr/bin/python

## adapted from <NAME>'s code. it can also be used with alignment sequences with models, e.g., rdp alignment files. Wild card "*" could be used with for multiple input sequence files. 

from Bio import SeqIO
import numpy
import sys
import re
import gzip

def main(mode, fasta_files):
	num_seqs = 0
	lengths = []

	for fasta_file in fasta_files:
		for seq in SeqIO.parse(open(fasta_file), "fastq"):
			if seq.id[0] == "#":
				continue

			length = 0
			seq_string = str(seq.seq)
			if mode == "model":
				length = len(re.sub(non_model_regex, "", seq_string))
{'fraction_non_alphanumeric': 0.11325301204819277, 'fraction_numerical': 0.004819277108433735, 'mean_word_length': 3.905511811023622, 'pattern_counts': {'":': 4, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5792874', 'n_tokens_mistral': 464, 'n_tokens_neox': 423, 'n_words': 143}