<commit_before>#! /usr/bin/env python
import sys
from screed.fasta import fasta_iter

MAX_SIZE=50000
THRESHOLD=1

def read_partition_file(fp):
    for n, record in enumerate(fasta_iter(fp, parse_description=False)):
        name = record['name']
        name, partition_id = name.rsplit('\t', 1)
        yield n, name, partition_id, record['sequence']

###

(filename, prefix) = sys.argv[1:]

distfilename = prefix + '.dist'
distfp = open(distfilename, 'w')

print '---'
print 'reading partitioned file:', filename
print 'outputting to files named "%s.groupN.fa"' % prefix
print 'partition size distribution will go to %s' % distfilename
print '---'

###

count = {}

###

for n, name, pid, seq in read_partition_file(open(filename)):
    if n % 100000 == 0:
        print '...', n

    count[pid] = count.get(pid, 0) + 1

if 0 in count:                          # eliminate unpartitioned sequences
   del count[0]
   
# develop histogram of partition sizes
dist = {}
for pid, size in count.items():
    dist[size] = dist.get(size, 0) + 1
        
# output histogram
total = 0
for c, n in sorted(dist.items()):
    total += n
    distfp.write('%d %d %d\n' % (c, n, total))
distfp.close()

# sort groups by size
divvy = sorted(count.items(), key=lambda y:y[1])
divvy = filter(lambda y:y[1] > THRESHOLD, divvy)

## divvy up into different groups, based on having MAX_SIZE sequences
## in each group.
total = 0
group = set()<commit_msg>added cumulative sequences to extract-partitions dist output<commit_after>#! /usr/bin/env python
import sys
from screed.fasta import fasta_iter

MAX_SIZE=50000
THRESHOLD=1

def read_partition_file(fp):
    for n, record in enumerate(fasta_iter(fp, parse_description=False)):
        name = record['name']
        name, partition_id = name.rsplit('\t', 1)
        yield n, name, partition_id, record['sequence']

###

(filename, prefix) = sys.argv[1:]

distfilename = prefix + '.dist'
distfp = open(distfilename, 'w')

print '---'
print 'reading partitioned file:', filename
print 'outputting to files named "%s.groupN.fa"' % prefix
print 'partition size distribution will go to %s' % distfilename
print '---'

###

count = {}

###

for n, name, pid, seq in read_partition_file(open(filename)):
    if n % 100000 == 0:
        print '...', n

    count[pid] = count.get(pid, 0) + 1

if 0 in count:                          # eliminate unpartitioned sequences
   del count[0]
   
# develop histogram of partition sizes
dist = {}
for pid, size in count.items():
    dist[size] = dist.get(size, 0) + 1
        
# output histogram
total = 0
wtotal = 0
for c, n in sorted(dist.items()):
    total += n
    wtotal += c*n
    distfp.write('%d %d %d %d\n' % (c, n, total, wtotal))
distfp.close()

# sort groups by size
divvy = sorted(count.items(), key=lambda y:y[1])
divvy = filter(lambda y:y[1] > THRESHOLD, divvy)

## divvy up into different groups, based on having MAX_SIZE sequences
## in each group.
total = 0
group = set()
{'fraction_non_alphanumeric': 0.14121232644768034, 'fraction_numerical': 0.017270572299356586, 'mean_word_length': 3.3698224852071004, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7123896', 'n_tokens_mistral': 1036, 'n_tokens_neox': 985, 'n_words': 315}