<commit_before>import khmer
import sys
import screed

K = 32
HASHTABLE_SIZE=int(256e9)
THRESHOLD=100

print 'creating ht'
ht = khmer.new_hashbits(K, HASHTABLE_SIZE, 2)

for filename in sys.argv[1:]:
    print 'eating fa', filename
    total_reads, n_consumed = ht.consume_fasta(filename)

#ht.save(sys.argv[1] + '.graphsize.ht')

for filename in sys.argv[1:]:
    outfp = open(filename + '.graphsize', 'w')

    n_kept = 0
    for n, record in enumerate(screed.fastq.fastq_iter(open(filename))):
        kmer = record['sequence'][:K]
        size = ht.calc_connected_graph_size(kmer, THRESHOLD)
        if size >= THRESHOLD:
            print >>outfp, ">%s\n%s" % (record['name'], record['sequence'])
            n_kept += 1

        if n % 10000 == 0:
            print '...', n, n_kept, n - n_kept

    <commit_msg>added output to graph-size-py.py<commit_after>import khmer
import sys
import screed
import glob
import os
import subprocess
import zlib
import gzip

K = 32
HASHTABLE_SIZE=4**17+1
THRESHOLD=100

print 'creating ht with size ' + str(HASHTABLE_SIZE)
ht = khmer.new_hashbits(K, HASHTABLE_SIZE, 1)

read_count = 0

fd = open(sys.argv[1], 'w')

files = glob.glob(sys.argv[2])

for filename in files:
    outfp = open(filename + '.graphsize', 'w')
    
    print 'processing file: ' + filename + ' reads processed: ' + str(read_count)
    n_kept = 0
    for n, record in enumerate(screed.fastq.fastq_iter(gzip.open(filename))):
        read_count += 1
        ht.consume(record['sequence'])

        if read_count % 10000 == 0:
            fd.write(str(read_count) + " " + str(ht.n_occupied()) + " " + str(ht.n_occupied() / float(HASHTABLE_SIZE)) +'\n')
            fd.flush()

        kmer = record['sequence'][:K]
        size = ht.calc_connected_graph_size(kmer, THRESHOLD)
        if size >= THRESHOLD:
            print >>outfp, ">%s\n%s" % (record['name'], record['sequence'])
            n_kept += 1

        if n % 10000 == 0:
            print '...', n, n_kept, n - n_kept + 1
{'fraction_non_alphanumeric': 0.14636591478696742, 'fraction_numerical': 0.02506265664160401, 'mean_word_length': 2.9137254901960783, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 11, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6056283', 'n_tokens_mistral': 784, 'n_tokens_neox': 727, 'n_words': 169}