<commit_before>import sys, logging, os, re
import edl.util, edl.taxon, edl.silva

logging.basicConfig(stream=sys.stdout)
logger=logging.getLogger(sys.argv[0])
logger.setLevel(logging.INFO)
logger.setLevel(logging.DEBUG)

## Some ranks need to be renamed to work with existing scripts
# superkingdom->major_clade: NCBI uses superkingdom for domain, so this has to be changed to a reasonable alternative
rankMapping={"superkingdom":"major_clade"}

def main():
    # command line arguments (this could be more robust...)
    logger.debug(sys.argv)
    (taxfile, fastafile, dumpDir) = sys.argv[1:]<commit_msg>Silva looks good<commit_after>import sys, logging, os, re
import edl.util, edl.taxon, edl.silva

logging.basicConfig(stream=sys.stdout)
logger=logging.getLogger(sys.argv[0])
logger.setLevel(logging.INFO)
#logger.setLevel(logging.DEBUG)

## Some ranks need to be renamed to work with existing scripts
# superkingdom->major_clade: NCBI uses superkingdom for domain, so this has to be changed to a reasonable alternative
rankMapping={"superkingdom":"major_clade"}

def main():
    # command line arguments (this could be more robust...)
    logger.debug(sys.argv)
    (taxfile, fastafile, dumpDir) = sys.argv[1:]
{'fraction_non_alphanumeric': 0.1128500823723229, 'fraction_numerical': 0.0032948929159802307, 'mean_word_length': 5.497326203208556, 'pattern_counts': {'":': 2, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5144335', 'n_tokens_mistral': 380, 'n_tokens_neox': 376, 'n_words': 113}